hadupils 0.4.0 → 0.5.0

data/CHANGELOG.md

@@ -44,3 +44,15 @@
   Helper method for assembling gzipped archives the contents of which
   are hive-ext compatible.
 
+### 0.5.0
+* Introduced a hadoop command enforcing user config and hadoop-ext
+  configuration files (hadoop.xml) that calls the hadoop runner
+* Introduced the following commands and runners that utilize the hadoop
+  runner (to work against the DFS): mktemp, withtmpdir and rm
+* Support for hadoop_confs in Hadupils::Extensions
+* Introduced Hadupils::Extensions::Dfs::TmpFile
+* Introduced Hadupils::Hacks module for String Refinements (self.randcase)
+  for Ruby 2+ and Monkey Patching for the String class for Ruby < 2.0
+* Some refactoring and fixed a bug with the specs for Mac OS X
+* Tweaked old unit tests and added new ones for the new features
+* Updated the README with examples

data/Gemfile

@@ -1,6 +1,13 @@
-source :rubygems
+source 'https://rubygems.org'
 
-gem 'mocha',            :group => :test
-gem 'rake', '10.1.0',   :group => :development
-gem 'shoulda-context',  :group => :test
+gem 'uuid',              '~> 2.3.0'
 
+group :development do
+  gem 'rake',              '~> 10.1.0'
+  gem 'pry',               '~> 0.9.0'
+end
+
+group :test do
+  gem 'mocha',                        :group => :test
+  gem 'shoulda-context',              :group => :test
+end

data/Gemfile.lock

@@ -1,16 +1,30 @@
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
+    coderay (1.0.9)
+    macaddr (1.6.1)
+      systemu (~> 2.5.0)
     metaclass (0.0.1)
+    method_source (0.8.1)
     mocha (0.14.0)
       metaclass (~> 0.0.1)
+    pry (0.9.12)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
     rake (10.1.0)
     shoulda-context (1.1.5)
+    slop (3.4.4)
+    systemu (2.5.2)
+    uuid (2.3.7)
+      macaddr (~> 1.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   mocha
-  rake (= 10.1.0)
+  pry (~> 0.9.0)
+  rake (~> 10.1.0)
   shoulda-context
+  uuid (~> 2.3.0)

data/README.md

@@ -2,3 +2,28 @@ hadupils
 ========
 
 Operating environment oriented utilities for hadoop (Hadoop + Utils => hadupils)
+
+## Shell Environment Variables
+- $HADUPILS_BASE_TMP_PATH
+    * This is the base path for DFS temporary file/directory creation
+    * Defaults to '/tmp' on the DFS (only set this if you need another base directory)
+- $HADUPILS_TMPDIR_PATH
+    * Set when the subcommand is executed in a subshell via the hadupils 'withtmpdir' command
+    * The value comes from the tmp directory that hadupils created for the subcommand
+    * It will cleanup (remove) the directory if the subcommand returns an exitstatus of zero
+
+## Hadpuils' Commands
+- hive __command__ _options_
+- hadoop __command__ _options_
+- mktemp [-d]
+- withtmpdir __subshell_command__
+- rm [-r] __full_path_to_file_or_directory__
+
+### Example Usages
+``` shell
+hadupils hive -e 'select a.col from tab1 a'
+hadupils hadoop fs -ls /tmp
+hadupils mktemp -d
+hadupils withtmpdir 'echo $HADUPILS_TMPDIR_PATH'
+hadupils rm -r /tmp/hadupils-tmp-e341afe01721013128c122000af92329
+```

data/bin/hadupils

@@ -5,4 +5,3 @@
 require 'hadupils'
 
 exit Hadupils::Commands.run ARGV[0], ARGV[1..-1]
-

data/lib/hadupils/commands.rb

@@ -1,5 +1,5 @@
 module Hadupils::Commands
-  def self.run(command, params)
+  def self.run(command, params=[])
     handler = handler_for command
     handler.run params
   end
@@ -18,9 +18,13 @@ module Hadupils::Commands
   end
 
   class SimpleCommand
-    def self.run(params)
+    def self.run(params=[])
       self.new.run params
     end
+
+    def successful?(exitstatus)
+      exitstatus == 0
+    end
   end
 
   module HadoopExt
@@ -29,18 +33,42 @@ module Hadupils::Commands
     end
   end
 
+  module HiveExt
+    def hive_ext
+      @hive_ext ||= Hadupils::Extensions::HiveSet.new(Hadupils::Search.hive_extensions)
+    end
+  end
+
   module UserConf
     def user_config
       @user_config ||= Hadupils::Extensions::Static.new(Hadupils::Search.user_config)
     end
   end
 
-  module HiveExt
-    def hive_ext
-      @hive_ext ||= Hadupils::Extensions::HiveSet.new(Hadupils::Search.hive_extensions)
+  class Hadoop < SimpleCommand
+    include HadoopExt
+    include UserConf
+
+    def assemble_parameters(parameters)
+      @hadoop_ext = Hadupils::Extensions::Static.new(Hadupils::Search.hadoop_assets)
+      hadoop_cmd      = parameters[0...1]
+      hadoop_cmd_opts  = parameters[1..-1] || []
+
+      if %w(fs dfs).include? parameters[0]
+        hadoop_cmd + user_config.hadoop_confs + hadoop_ext.hadoop_confs + hadoop_cmd_opts
+      else
+        # TODO: Assemble command line parameters to pkg assets/code for submitting jobs, for i.e.
+        hadoop_cmd + user_config.hadoop_confs + hadoop_ext.hadoop_confs + hadoop_cmd_opts
+      end
+    end
+
+    def run(parameters)
+      Hadupils::Runners::Hadoop.run assemble_parameters(parameters)
     end
   end
 
+  register_handler :hadoop, Hadoop
+
   class Hive < SimpleCommand
     include HadoopExt
     include HiveExt
@@ -56,4 +84,88 @@ module Hadupils::Commands
   end
 
   register_handler :hive, Hive
+
+  class MkTmpFile < SimpleCommand
+    def run(parameters)
+      # Creates a new tmpdir and puts the full tmpdir_path to STDOUT
+      Hadupils::Extensions::Dfs::TmpFile.reset_tmpfile!
+      tmpdir_path = Hadupils::Extensions::Dfs::TmpFile.tmpfile_path
+
+      # Similar to shell mktemp, but for Hadoop DFS!
+      # Makes a tmp file by default; a tmp directory with '-d' flag
+      fs_cmd = parameters[0] == '-d' ? '-mkdir' : '-touchz'
+      exitstatus = Hadupils::Commands::Hadoop.run ['fs', fs_cmd, tmpdir_path]
+      if successful? exitstatus
+        exitstatus = Hadupils::Commands::Hadoop.run ['fs', '-chmod', '700', tmpdir_path]
+        if successful? exitstatus
+          puts tmpdir_path
+        else
+          $stderr.puts "Failed to chmod 700 dfs tmpdir: #{tmpdir_path}"
+        end
+      else
+        $stderr.puts "Failed creating dfs tmpdir: #{tmpdir_path}"
+      end
+      exitstatus
+    end
+  end
+
+  register_handler :mktemp, MkTmpFile
+
+  class RmFile < SimpleCommand
+    def run(parameters)
+      # Similar to shell rm, but for Hadoop DFS!
+      # Removes files by default; removes directories recursively with '-r' flag
+      fs_cmd, tmp_dirs =
+        if parameters[0] == '-r'
+          ['-rmr', parameters[1..-1]]
+        else
+          ['-rm', parameters[0..-1]]
+        end
+
+      if tmp_dirs.empty?
+        $stderr.puts 'Failed to remove unspecified tmpdir(s), please specify tmpdir_path'
+        255
+      else
+        exitstatus = Hadupils::Commands::Hadoop.run ['fs', fs_cmd, tmp_dirs].flatten
+        if successful? exitstatus
+          Hadupils::Extensions::Dfs::TmpFile.reset_tmpfile!
+        else
+          $stderr.puts "Failed to remove dfs tmpdir: #{tmp_dirs.join(' ')}"
+        end
+        exitstatus
+      end
+    end
+  end
+
+  register_handler :rm, RmFile
+
+  class WithTmpDir < SimpleCommand
+    def run(parameters)
+      # Runs provided subcommand with tmpdir and cleans up tmpdir on an exitstatus of zero
+      if parameters.empty?
+        $stderr.puts 'Yeeaaahhh... sooo... you failed to provide a subcommand...'
+        255
+      else
+        # Let's create the tmpdir
+        exitstatus = Hadupils::Commands::MkTmpFile.run ['-d']
+        if successful? exitstatus
+          tmpdir_path = Hadupils::Extensions::Dfs::TmpFile.tmpfile_path
+          parameters.unshift({'HADUPILS_TMPDIR_PATH' => tmpdir_path})
+
+          # Let's run the shell subcommand!
+          exitstatus = Hadupils::Runners::Subcommand.run parameters
+
+          if successful? exitstatus
+            # Let's attempt to cleanup tmpdir_path
+            exitstatus = Hadupils::Commands::RmFile.run ['-r', tmpdir_path]
+          else
+            $stderr.puts "Failed to run shell subcommand: #{parameters}"
+          end
+        end
+        exitstatus
+      end
+    end
+  end
+
+  register_handler :withtmpdir, WithTmpDir
 end

data/lib/hadupils/extensions/hive.rb

@@ -1,3 +1,6 @@
+require 'tempfile'
+require 'fileutils'
+
 module Hadupils::Extensions
 
   # Hive-targeted extensions derived from filesystem layout
@@ -204,8 +207,6 @@ module Hadupils::Extensions
         end
       end
 
-      require 'tempfile'
-      require 'fileutils'
       ::Dir.mktmpdir do |workdir|
         basenames = dist.collect do |src|
           FileUtils.cp src, File.join(workdir, File.basename(src))

data/lib/hadupils/extensions.rb

@@ -1,4 +1,55 @@
+require 'uuid'
+require 'tempfile'
+
 module Hadupils::Extensions
+  # Tools for managing tmp files in the hadoop dfs
+  module Dfs
+    module TmpFile
+      def self.uuid
+        @uuid ||= UUID.new
+      end
+
+      def self.tmp_path
+        @tmp_path ||= (ENV['HADUPILS_BASE_TMP_PATH'] || '/tmp')
+      end
+
+      def self.tmpfile_path
+        @tmpdir_path ||= ::File.join(tmp_path, "hadupils-tmp-#{uuid.generate(:compact)}")
+      end
+
+      def self.reset_tmpfile!
+        @tmpdir_path = nil
+      end
+    end
+  end
+
+  # Tools for managing hadoop configuration files ("hadoop.xml").
+  module HadoopConf
+    module HadoopOpt
+      def hadoop_opts
+        ['-conf', path]
+      end
+    end
+
+    # Wraps an extant hadoop configuration file and provides
+    # an interface compatible with the critical parts of the
+    # Static sibling class so they may be used interchangeably
+    # by runners when determining hadoop options.
+    class Static
+      attr_reader :path
+
+      include HadoopOpt
+
+      # Given a path, expands it ti
+      def initialize(path)
+        @path = ::File.expand_path(path)
+      end
+
+      def close
+      end
+    end
+  end
+
   # Tools for managing hive initialization files ("hiverc").
   module HiveRC
     module HiveOpt
@@ -32,7 +83,6 @@ module Hadupils::Extensions
       attr_reader :file
 
       include HiveOpt
-      require 'tempfile'
 
       # This will allow us to change what handles the dynamic files.
       def self.file_handler=(handler)
@@ -87,6 +137,12 @@ module Hadupils::Extensions
       end
     end
 
+    def hadoop_conf(&block)
+      @scope.instance_eval do
+        @hadoop_conf_block = block
+      end
+    end
+
     def hiverc(&block)
       @scope.instance_eval do
         @hiverc_block = block
@@ -110,12 +166,16 @@ module Hadupils::Extensions
       assets
     end
 
+    def hadoop_confs
+      []
+    end
+
     def hivercs
       []
     end
 
     def self.gather_assets(directory)
-      if not directory.nil?
+      if directory
         Hadupils::Assets.assets_in(directory)
       else
         []
@@ -179,12 +239,26 @@ module Hadupils::Extensions
       []
     end
 
+    def hadoop_conf_path
+      ::File.join(path, 'hadoop.xml') if path
+    end
+
     def hiverc_path
-      ::File.join(path, 'hiverc')
+      ::File.join(path, 'hiverc') if path
+    end
+
+    def hadoop_conf?
+      hadoop_conf_path ? ::File.file?(hadoop_conf_path) : false
     end
 
     def hiverc?
-      ::File.file? hiverc_path
+      hiverc_path ? ::File.file?(hiverc_path) : false
+    end
+
+    def hadoop_confs
+      r = []
+      r << Hadupils::Extensions::HadoopConf::Static.new(hadoop_conf_path) if hadoop_conf?
+      r
     end
 
     def hivercs

data/lib/hadupils/hacks.rb

@@ -0,0 +1,29 @@
+if RUBY_VERSION < '2.0'
+  class String
+    def randcase
+      dup.split('').map do |char|
+        if rand(1..10) > 5
+          char.upcase
+        else
+          char.downcase
+        end
+      end.join
+    end
+  end
+else
+  module Hadupils
+    module Hacks
+      refine ::String do
+        def randcase
+          dup.split('').map do |char|
+            if rand(1..100) > 50
+              char.upcase
+            else
+              char.downcase
+            end
+          end.join
+        end
+      end
+    end
+  end
+end

data/lib/hadupils/runners.rb

@@ -6,6 +6,10 @@ module Hadupils::Runners
       @params = params
     end
 
+    def self.run(*params)
+      self.new(*params).wait!
+    end
+
     def command; end
 
     def execute!
@@ -35,19 +39,39 @@ module Hadupils::Runners
     def wait!
       @last_result = execute!
       @last_status = $?
+      last_exitstatus
+    end
+
+    def last_exitstatus
       if @last_result.nil?
         255
       else
         @last_status.exitstatus
       end
     end
+  end
 
-    def self.run(*params)
-      self.new(*params).wait!
+  class Hadoop < Base
+    class << self; attr_writer :base_runner; end
+
+    def self.base_runner
+      @base_runner || ::File.join(ENV['HADOOP_HOME'], 'bin', 'hadoop')
+    end
+
+    def command
+      params.inject([self.class.base_runner]) do |result, param|
+        if param.respond_to? :hadoop_opts
+          result + param.hadoop_opts
+        else
+          result << param
+        end
+      end
     end
   end
 
   class Hive < Base
+    class << self; attr_writer :base_runner; end
+
     def initialize(params, hive_aux_jars_path='')
       super(params)
       @hive_aux_jars_path = hive_aux_jars_path
@@ -57,10 +81,6 @@ module Hadupils::Runners
       @base_runner || ::File.join(ENV['HIVE_HOME'], 'bin', 'hive')
     end
 
-    def self.base_runner=(runner_path)
-      @base_runner = runner_path
-    end
-
     def command
       params.inject([env_overrides, self.class.base_runner]) do |result, param|
         if param.respond_to? :hive_opts
@@ -72,14 +92,20 @@ module Hadupils::Runners
     end
 
     def env_overrides
-      e = {}
+      env = {}
       settings = [@hive_aux_jars_path, ::ENV['HIVE_AUX_JARS_PATH']].reject do |val|
-        val.nil? or val.strip == ''
+        val.nil? || val.strip.empty?
       end
       if settings.length > 0
-        e['HIVE_AUX_JARS_PATH'] = settings.join(',')
+        env['HIVE_AUX_JARS_PATH'] = settings.join(',')
       end
-      e
+      env
+    end
+  end
+
+  class Subcommand < Base
+    def command
+      params
     end
   end
 end

data/lib/hadupils/util.rb

@@ -1,7 +1,8 @@
+require 'rubygems/package'
+require 'zlib'
+
 module Hadupils::Util
   def self.read_archive(archive_path)
-    require 'rubygems/package'
-    require 'zlib'
     Zlib::GzipReader.open(archive_path) do |zlib|
       Gem::Package::TarReader.new(zlib) do |tar|
         tar.rewind

data/lib/hadupils.rb

@@ -8,3 +8,4 @@ require 'hadupils/extensions'
 require 'hadupils/runners'
 require 'hadupils/search'
 require 'hadupils/util'
+require 'hadupils/hacks'

data/test/hadupil_test_setup.rb

@@ -4,6 +4,7 @@ require 'test/unit'
 require 'shoulda-context'
 require 'mocha/setup'
 require 'tempfile'
+require 'pathname'
 require 'hadupils'
 
 # Add tempdir niceties to Test::Unit::TestCase
@@ -35,7 +36,7 @@ class Test::Unit::TestCase
   def self.tempdir_context(name, &block)
     context name do
       setup do
-        @tempdir = Test::Unit::TestCase::DirWrapper.new(::File.expand_path(::Dir.mktmpdir))
+        @tempdir = Test::Unit::TestCase::DirWrapper.new(::Pathname.new(::File.expand_path(::Dir.mktmpdir)).realpath.to_s)
       end
 
       teardown do

data/test/unit/commands_test.rb

@@ -1,3 +1,5 @@
+using Hadupils::Hacks unless RUBY_VERSION < '2.0'
+
 class Hadupils::CommandsTest < Test::Unit::TestCase
   context Hadupils::Commands do
     context 'run singleton method' do
@@ -26,16 +28,100 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
       end
     end
 
+    context 'Hadoop' do
+      setup do
+        @klass = Hadupils::Commands::Hadoop
+      end
+
+      should 'register with :hadoop name' do
+        handlers = [:hadoop]
+        run_handler_assertions_for handlers
+      end
+
+      should 'have a #run singleton method that dispatches to an instance #run' do
+        @klass.expects(:new).with.returns(instance = mock())
+        instance.expects(:run).with(params = mock()).returns(result = mock())
+        assert_equal result, @klass.run(params)
+      end
+
+      should 'have a Static extension based on a search for hadoop-ext' do
+        Hadupils::Search.expects(:hadoop_assets).with.returns(conf = mock())
+        Hadupils::Extensions::Static.expects(:new).with(conf).returns(extension = mock())
+        hadoop_ext = Hadupils::Extensions::Static.new(Hadupils::Search.hadoop_assets)
+        cmd = @klass.new
+        cmd.stubs(:hadoop_ext).with.returns(hadoop_ext)
+        assert_equal extension, cmd.hadoop_ext
+        # This should cause failure if the previous result wasn't
+        # cached internally (by breaking expectations).
+        cmd.hadoop_ext
+      end
+
+      should 'have a Static extensions based on user config' do
+        Hadupils::Search.expects(:user_config).with.returns(conf = mock())
+        Hadupils::Extensions::Static.expects(:new).with(conf).returns(extension = mock())
+        cmd = @klass.new
+        assert_equal extension, cmd.user_config
+        # Fails on expectations if previous result wasn't cached.
+        cmd.user_config
+      end
+
+      context '#run' do
+        setup do
+          @command = @klass.new
+          @command.stubs(:user_config).with.returns(@user_config = mock())
+          @command.stubs(:hadoop_ext).with.returns(@hadoop_ext = mock())
+          @runner_class = Hadupils::Runners::Hadoop
+        end
+
+        context 'with user config and hadoop_confs' do
+          setup do
+            @user_config.stubs(:hadoop_confs).returns(@user_config_hadoop_confs = [mock(), mock()])
+            @hadoop_ext.stubs(:hadoop_confs).returns(@hadoop_ext_hadoop_confs = [mock(), mock(), mock()])
+          end
+
+          should 'apply hadoop_conf options to hadoop runner call' do
+            @runner_class.expects(:run).with(@user_config_hadoop_confs +
+                                             @hadoop_ext_hadoop_confs).returns(result = mock())
+            assert_equal result, @command.run([])
+          end
+
+          should 'insert hadoop_conf options into position 1 of given params array to hadoop runner call' do
+            params = [mock(), mock()]
+            @runner_class.expects(:run).with(params[0...1] +
+                                             @user_config_hadoop_confs +
+                                             @hadoop_ext_hadoop_confs +
+                                             params[1..-1]).returns(result = mock())
+            assert_equal result, @command.run(params)
+          end
+        end
+
+        context 'without hadoop_confs' do
+          setup do
+            @user_config.stubs(:hadoop_confs).returns([])
+            @hadoop_ext.stubs(:hadoop_confs).returns([])
+          end
+
+          should 'pass params unchanged through to hadoop runner call' do
+            @runner_class.expects(:run).with(params = [mock(), mock()]).returns(result = mock())
+            assert_equal result, @command.run(params)
+          end
+
+          should 'handle empty params' do
+            @runner_class.expects(:run).with([]).returns(result = mock())
+            assert_equal result, @command.run([])
+          end
+        end
+      end
+    end
+
     context 'Hive' do
       setup do
         @klass = Hadupils::Commands::Hive
       end
 
       should 'register with :hive name' do
-        assert_same @klass, Hadupils::Commands.handler_for(:hive)
-        assert_same @klass, Hadupils::Commands.handler_for(:HivE)
-        assert_same @klass, Hadupils::Commands.handler_for('hive')
-        assert_same @klass, Hadupils::Commands.handler_for('hIVe')
+        handlers = [:hive]
+        run_handler_assertions_for handlers
       end
 
       should 'have a #run singleton method that dispatches to an instance #run' do
@@ -207,21 +293,21 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
 
         should 'produce a valid set of parameters and hivercs' do
           Kernel.stubs(:system).with() do |*args|
-            args[0] == {'HIVE_AUX_JARS_PATH' => @hive_aux_jars_path_val} and
-            args[1] == @hive_prog and
-            args[2] == '-i' and
-            File.open(args[3], 'r').read == @static_hiverc_content and
-            args[4] == '-i' and
-            File.open(args[5], 'r').read == @dynamic_hiverc_content and
-            args[6] == '-i' and
-            File.open(args[7], 'r').read == @hive_exts[:one][:dynamic_hiverc_content] and
-            args[8] == '-i' and
-            File.open(args[9], 'r').read == @hive_exts[:one][:static_hiverc_content] and
-            args[10] == '-i' and
-            File.open(args[11], 'r').read == @hive_exts[:two][:dynamic_hiverc_content] and
-            args[12] == '-i' and
-            File.open(args[13], 'r').read == @hive_exts[:two][:static_hiverc_content] and
-            args[14] == '--hiveconf' and
+            args[0] == {'HIVE_AUX_JARS_PATH' => @hive_aux_jars_path_val} &&
+            args[1] == @hive_prog &&
+            args[2] == '-i' &&
+            File.open(args[3], 'r').read == @static_hiverc_content &&
+            args[4] == '-i' &&
+            File.open(args[5], 'r').read == @dynamic_hiverc_content &&
+            args[6] == '-i' &&
+            File.open(args[7], 'r').read == @hive_exts[:one][:dynamic_hiverc_content] &&
+            args[8] == '-i' &&
+            File.open(args[9], 'r').read == @hive_exts[:one][:static_hiverc_content] &&
+            args[10] == '-i' &&
+            File.open(args[11], 'r').read == @hive_exts[:two][:dynamic_hiverc_content] &&
+            args[12] == '-i' &&
+            File.open(args[13], 'r').read == @hive_exts[:two][:static_hiverc_content] &&
+            args[14] == '--hiveconf' &&
             args[15] == 'my.foo=your.fu'
           end
           Hadupils::Commands.run 'hive', ['--hiveconf', 'my.foo=your.fu']
@@ -231,7 +317,149 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
           ::Dir.chdir @pwd
         end
       end
+      context 'MkTempFile' do
+        setup do
+          @klass = Hadupils::Commands::MkTmpFile
+        end
+
+        should 'register with :mktemp name' do
+          handlers = [:mktemp]
+          run_handler_assertions_for handlers
+        end
+
+        should 'have a #run singleton method that dispatches to an instance #run' do
+          @klass.expects(:new).with.returns(instance = mock())
+          instance.expects(:run).with(params = mock()).returns(result = mock())
+          assert_equal result, @klass.run(params)
+        end
+
+        context '#run' do
+          setup do
+            @command = @klass.new
+            Hadupils::Runners::Hadoop.stubs(:base_runner).returns(@hadoop_path = mock().to_s + '-hadoop')
+          end
+
+          should 'provide invocation for bare mktemp if given empty parameters' do
+            tmpdir_path = mock().to_s
+            Hadupils::Extensions::Dfs::TmpFile.expects(:tmpfile_path).returns(tmpdir_path)
+            Kernel.expects(:system).with(@hadoop_path, 'fs', '-touchz', tmpdir_path).returns(0)
+            Kernel.expects(:system).with(@hadoop_path, 'fs', '-chmod', '700', tmpdir_path).returns(0)
+            assert_equal 0, @command.run([])
+          end
+
+          should 'provide invocation for mktemp if given with -d flag parameter' do
+            tmpdir_path = mock().to_s
+            Hadupils::Extensions::Dfs::TmpFile.expects(:tmpfile_path).returns(tmpdir_path)
+            Kernel.expects(:system).with(@hadoop_path, 'fs', '-mkdir', tmpdir_path).returns(0)
+            Kernel.expects(:system).with(@hadoop_path, 'fs', '-chmod', '700', tmpdir_path).returns(0)
+            assert_equal 0, @command.run(['-d'])
+          end
+        end
+      end
+
+      context 'RmFile' do
+        setup do
+          @klass = Hadupils::Commands::RmFile
+        end
+
+        should 'register with :rm name' do
+          handlers = [:rm]
+          run_handler_assertions_for handlers
+        end
+
+        should 'have a #run singleton method that dispatches to an instance #run' do
+          @klass.expects(:new).with.returns(instance = mock())
+          instance.expects(:run).with(params = mock()).returns(result = mock())
+          assert_equal result, @klass.run(params)
+        end
+
+        context '#run' do
+          setup do
+            @command = @klass.new
+            Hadupils::Runners::Hadoop.stubs(:base_runner).returns(@hadoop_path = mock().to_s + '-hadoop')
+          end
+
+          should 'provide invocation for bare rm if given empty parameters' do
+            assert_equal 255, @klass.run([])
+          end
+
+          should 'provide invocation for rm if just tmpdir_path parameter' do
+            tmpdir_path = mock().to_s
+            Kernel.expects(:system).with(@hadoop_path, 'fs', '-rm', tmpdir_path).returns(0)
+            assert_equal 0, @klass.run([tmpdir_path])
+          end
+
+          should 'provide invocation for hadoop if just tmpdir_path with -r flag parameter' do
+            tmpdir_path = mock().to_s
+            Kernel.expects(:system).with(@hadoop_path, 'fs', '-rmr', tmpdir_path).returns(0)
+            assert_equal 0, @klass.run(['-r', tmpdir_path])
+          end
+        end
+
+        context 'WithTempDir' do
+          setup do
+            @klass = Hadupils::Commands::WithTmpDir
+          end
+
+          should 'register with :withtmpdir name' do
+            handlers = [:withtmpdir]
+            run_handler_assertions_for handlers
+          end
+
+          should 'have a #run singleton method that dispatches to an instance #run' do
+            @klass.expects(:new).with.returns(instance = mock())
+            instance.expects(:run).with(params = mock()).returns(result = mock())
+            assert_equal result, @klass.run(params)
+          end
+
+          context '#run' do
+            setup do
+              @command = @klass.new
+              Hadupils::Runners::Hadoop.stubs(:base_runner).returns(@hadoop_path = mock().to_s + '-hadoop')
+            end
+
+            should 'provide invocation for withtmpdir if given parameters for shell subcommand' do
+              tmpdir_path = mock().to_s
+              run_common_subcommand_assertions_with tmpdir_path
+              Kernel.expects(:system).with({'HADUPILS_TMPDIR_PATH' => tmpdir_path}, '/path/to/my_wonderful_script.sh').returns(0)
+              Kernel.expects(:system).with(@hadoop_path, 'fs', '-rmr', tmpdir_path).returns(0)
+              assert_equal 0, @klass.run(['/path/to/my_wonderful_script.sh'])
+            end
+
+            should 'provide invocation for withtmpdir if given parameters for shell subcommand (another hadupils command)' do
+              tmpdir_path = mock().to_s
+              run_common_subcommand_assertions_with tmpdir_path
+              Kernel.expects(:system).with({'HADUPILS_TMPDIR_PATH' => tmpdir_path}, 'hadupils hadoop ls /tmp').returns(0)
+              Kernel.expects(:system).with(@hadoop_path, 'fs', '-rmr', tmpdir_path).returns('')
+              assert_equal 0, @klass.run(['hadupils hadoop ls /tmp'])
+            end
+
+            should 'provide invocation for withtmpdir if given parameters for shell subcommand with nil result' do
+              tmpdir_path = mock().to_s
+              run_common_subcommand_assertions_with tmpdir_path
+              Kernel.expects(:system).with({'HADUPILS_TMPDIR_PATH' => tmpdir_path}, '/path/to/my_wonderful_script.sh').returns(nil)
+              assert_equal 255, @klass.run(['/path/to/my_wonderful_script.sh'])
+            end
+          end
+        end
+      end
     end
   end
-end
 
+  def run_common_subcommand_assertions_with(tmpdir_path)
+    Hadupils::Extensions::Dfs::TmpFile.expects(:tmpfile_path).returns(tmpdir_path)
+    Hadupils::Extensions::Dfs::TmpFile.expects(:tmpfile_path).returns(tmpdir_path)
+    Kernel.expects(:system).with(@hadoop_path, 'fs', '-mkdir', tmpdir_path).returns(0)
+    Kernel.expects(:system).with(@hadoop_path, 'fs', '-chmod', '700', tmpdir_path).returns(0)
+  end
+
+  def run_handler_assertions_for(handlers)
+    handlers.each do |handler|
+      handler = handler.to_s.downcase
+      assert_same @klass, Hadupils::Commands.handler_for(handler.to_sym)
+      assert_same @klass, Hadupils::Commands.handler_for(handler.randcase.to_sym)
+      assert_same @klass, Hadupils::Commands.handler_for(handler)
+      assert_same @klass, Hadupils::Commands.handler_for(handler.randcase)
+    end
+  end
+end

data/test/unit/extensions/base_test.rb

@@ -1,3 +1,6 @@
+require 'rubygems/package'
+require 'zlib'
+
 class Hadupils::ExtensionsTest < Test::Unit::TestCase
   context Hadupils::Extensions::Base do
     context 'initialization with nil path' do
@@ -234,8 +237,6 @@ class Hadupils::ExtensionsTest < Test::Unit::TestCase
 
       context 'ArchivePath extension' do
         setup do
-          require 'rubygems/package'
-          require 'zlib'
           @klass = Hadupils::Extensions::FlatArchivePath
           @bin_archives = %w(one two).collect do |name|
             @tempdir.file("zarchive-with-bin-#{name}.tar.gz") do |f|

data/test/unit/extensions/hive_test.rb

@@ -1,3 +1,5 @@
+require 'stringio'
+
 class Hadupils::Extensions::HiveTest < Test::Unit::TestCase
   shared_context :provide_hive_ext do
     setup do
@@ -235,8 +237,6 @@ class Hadupils::Extensions::HiveTest < Test::Unit::TestCase
     end
 
     context 'build_archive singleton method' do
-      require 'stringio'
-
       setup do
         ::Dir.mkdir(@some_dir = @tempdir.full_path('some_stuff'))
         ::Dir.mkdir(@other_dir = @tempdir.full_path('other_stuff'))

data/test/unit/runners_test.rb

@@ -85,6 +85,49 @@ class Hadupils::RunnersTest < Test::Unit::TestCase
     end
   end
 
+  context Hadupils::Runners::Hadoop do
+    setup do
+      @klass = Hadupils::Runners::Hadoop
+    end
+
+    should 'be a runner' do
+      assert_kind_of Hadupils::Runners::Base, @klass.new([])
+    end
+
+    should 'use $HADOOP_HOME/bin/hadoop as the base runner' do
+      ENV.expects(:[]).with('HADOOP_HOME').returns(home = mock().to_s)
+      assert_equal ::File.join(home, 'bin', 'hadoop'),
+                   @klass.base_runner
+    end
+
+    context '#command' do
+      setup do
+        @klass.stubs(:base_runner).returns(@hadoop_path = mock().to_s + '-hadoop')
+      end
+
+      should 'provide invocation for bare hadoop if given empty parameters' do
+        assert_equal [@hadoop_path], @klass.new([]).command
+      end
+
+      should 'provide invocation for hadoop with all given parameters' do
+        params = [mock().to_s, mock().to_s, mock().to_s, mock().to_s]
+        assert_equal [@hadoop_path] + params,
+                     @klass.new(params).command
+      end
+
+      should 'provide args for hadoop with :hadoop_opts on supporting params' do
+        p1 = mock()
+        p1.expects(:hadoop_opts).with.returns(p1_opts = ['-conf', mock().to_s])
+        p2 = mock()
+        p2.expects(:hadoop_opts).with.returns(p2_opts = ['-conf', mock().to_s])
+        s1 = mock().to_s
+        s2 = mock().to_s
+        assert_equal [@hadoop_path, s1] + p1_opts + p2_opts + [s2],
+                     @klass.new([s1, p1, p2, s2]).command
+      end
+    end
+  end
+
   context Hadupils::Runners::Hive do
     setup do
       @klass = Hadupils::Runners::Hive

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hadupils
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,24 +9,40 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-22 00:00:00.000000000 Z
+date: 2013-10-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.5
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -48,17 +64,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 10.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 10.1.0
 - !ruby/object:Gem::Dependency
   name: shoulda-context
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +103,7 @@ files:
 - lib/hadupils/runners.rb
 - lib/hadupils/extensions/hive.rb
 - lib/hadupils/extensions.rb
+- lib/hadupils/hacks.rb
 - lib/hadupils/assets.rb
 - lib/hadupils/util.rb
 - lib/hadupils.rb