hadupils 0.1.3 → 0.2.0

data/CHANGELOG.md

@@ -16,3 +16,17 @@
 * Fixed embarrassing mispelling of "shoulda-context" in gemspec
   development dependencies
 
+### 0.1.3
+
+* Fixed compat. issue for ruby 1.8; downcase on symbol is a no-no.
+  In Hadupils::Command module.
+
+### 0.2.0
+
+* Introduced hive extensions (hive-ext)
+* Hive command uses hive extensions to assemble hivercs
+* Hive command assembles HIVE_AUX_JARS_PATH variable
+* Base runner can handle environment variable hash as first command
+  parameter (custom support for pre-1.9 ruby, the Kernel.system call
+  of which does not handle such things)
+

data/lib/hadupils/commands.rb

@@ -35,16 +35,23 @@ module Hadupils::Commands
     end
   end
 
+  module HiveExt
+    def hive_ext
+      @hive_ext ||= Hadupils::Extensions::HiveSet.new(Hadupils::Search.hive_extensions)
+    end
+  end
+
   class Hive < SimpleCommand
     include HadoopExt
+    include HiveExt
     include UserConf
 
     def assemble_parameters(parameters)
-      user_config.hivercs + hadoop_ext.hivercs + parameters
+      user_config.hivercs + hadoop_ext.hivercs + hive_ext.hivercs + parameters
     end
 
     def run(parameters)
-      Hadupils::Runners::Hive.run assemble_parameters(parameters)
+      Hadupils::Runners::Hive.run assemble_parameters(parameters), hive_ext.hive_aux_jars_path
     end
   end
 

data/lib/hadupils/extensions/hive.rb

@@ -0,0 +1,249 @@
+module Hadupils::Extensions
+
+  # Hive-targeted extensions derived from filesystem layout
+  #
+  # = Concept
+  # 
+  # There are a few ways to "extend" one's hive session:
+  #
+  # * Adding files, archives, jars to it (+ADD ...+).
+  # * Setting variables and whatnot (+SET ...+).
+  # * Registering your own UDFS.
+  # * Specifying paths to jars to make available within the session's
+  #   classpath (+HIVE_AUX_JARS_PATH+ env. var.).
+  #
+  # All of these things can be done through the use of initialization
+  # files (via hive's +-i+ option), except for the auxiliary jar libs
+  # environment variable (which is.... wait for it... in the environment).
+  #
+  # This class provides an abstraction to enable the following:
+  # * lay your files out according to its expectations
+  # * wrap that layout with an instance of this class
+  # * it'll give an interface for accessing initialization files (#hivercs)
+  #   that make the stuff available in a hive session
+  # * it'll dynamically assemble the initialization file necessary to
+  #   ensure appropriate assets are made available in the session
+  # * if you provide your own initialization file in the expected place,
+  #   it'll ensure that the dynamic stuff is applied _first_ and the static
+  #   one second, such that your static one can assume the neighboring
+  #   assets are already in the session.
+  # * it'll give you a list of jars to make available as auxiliary_jars in the
+  #   session based on contents of +aux-jars+.
+  # 
+  # You lay it down, the object makes sense of it, nothing other than
+  # file organization required.
+  #
+  # = Filesystem Layout
+  #
+  # Suppose you have the following stuff (denoting symlinks with +->+):
+  #
+  #     /etc/foo/
+  #         an.archive.tar.gz
+  #         another.archive.tar.gz
+  #         aux-jars/
+  #             aux-only.jar
+  #             ignored.archive.tar.gz
+  #             ignored.file.txt
+  #             jarry.jar -> ../jarry.jar
+  #         dist-only.jar
+  #         hiverc
+  #         jarry.jar
+  #         textie.txt
+  #         yummy.yaml
+  # 
+  # Now you create an instance:
+  #
+  #     ext = Hadupils::Extensions::Hive.new('/etc/foo')
+  #
+  # You could get the hive command-line options for using this stuff
+  # via:
+  #
+  #     ext.hivercs
+  #
+  # It'll give you objects for two initialization files:
+  # 1. A dynamic one that has the appropriate commands for adding
+  #    +an.archive.tar.gz+, +another.archive.tar.gz+, +dist-only.jar+,
+  #    +jarry.jar+, +textie.txt+, and +yummy.yaml+ to the session.
+  # 2. The +hiverc+ one that's in there.
+  #
+  # And, the +ext.auxiliary_jars+ accessor will return a list of paths to
+  # the jars (_only_ the jars) contained within the +aux-jars+ path;
+  # a caller to hive would use this to construct the +HIVE_AUX_JARS_PATH+
+  # variable.
+  #
+  # Notice that +jarry.jar+ is common to the distributed usage (it'll be
+  # added to the session and associated distributed cache) and to the
+  # auxiliary path.  That's because it appears in the main directory and
+  # in the +aux-jars+ subdirectory.  There's nothing magical about the
+  # use of a symlink; that just saves disk space.  10 MB ought be enough
+  # for anyone.
+  #
+  # If there was no +hiverc+ file, then you would only get the
+  # initialization file object for the loading of assets in the main
+  # directory.  Conversely, if there were no such assets, but there was
+  # a +hiverc+ file, you would get only the object for that file.  If
+  # neither were present, the #hivercs will be an empty list.
+  #
+  # If there is no +aux-jars+ directory, or that directory has no jars,
+  # the +ext.auxiliary_jars+ would be an empty list.  Only jars will be included
+  # in that list; files without a +.jar+ extension will be ignored.
+  #
+  class Hive
+    module AuxJarsPath
+      # A string representation of the hive auxiliary jars paths,
+      # based on #auxiliary_jars, suitable for usage as the value
+      # of +HIVE_AUX_JARS_PATH+ within the environment.
+      def hive_aux_jars_path
+        auxiliary_jars.collect {|jar| jar.strip}.join(',')
+      end
+    end
+
+    include AuxJarsPath
+
+    AUX_PATH = 'aux-jars'
+    HIVERC_PATH = 'hiverc'
+
+    attr_reader :auxiliary_jars
+    attr_reader :path
+
+    def initialize(path)
+      @path = ::File.expand_path(path)
+      @auxiliary_jars = self.class.find_auxiliary_jars(@path)
+      @dynamic_ext = self.class.assemble_dynamic_extension(@path)
+      @static_ext = self.class.assemble_static_extension(@path)
+    end
+
+    # An array of hive initialization objects derived from
+    # dynamic and static sets.  May be an empty list.  Dynamic
+    # are guaranteed to come before static, so a static +hiverc+ can
+    # count on the other assets being available.
+    def hivercs
+      dynamic_hivercs + static_hivercs
+    end
+
+    # An array of dynamic, managed hive initialization objects
+    # (Hadupils::Extensions::HiveRC::Dynamic) based on the assets
+    # found within the #path.  May be an empty list.
+    def dynamic_hivercs
+      if @dynamic_ext.assets.length > 0
+        @dynamic_ext.hivercs
+      else
+        []
+      end
+    end
+    
+    # An array of static hive initialization objects
+    # (Hadupils::Extensions::HiveRC::Static) based on the presence
+    # of a +hiverc+ file within the #path.  May be an empty list.
+    def static_hivercs
+      @static_ext.hivercs
+    end
+
+    def self.find_auxiliary_jars(path)
+      target = ::File.join(path, AUX_PATH)
+      if ::File.directory? target
+        jars = Hadupils::Assets.assets_in(target).find_all do |asset|
+          asset.kind_of? Hadupils::Assets::Jar
+        end
+        jars.collect {|asset| asset.path}
+      else
+        []
+      end
+    end
+
+    def self.assemble_dynamic_extension(path)
+      Flat.new(path) do
+        assets do |list|
+          list.reject {|asset| [AUX_PATH, HIVERC_PATH].include? asset.name }
+        end
+      end
+    end
+
+    def self.assemble_static_extension(path)
+      Static.new(path)
+    end
+  end
+
+  # Collection class for filesystem-based Hive extensions
+  #
+  # Pretty simple:
+  # * Given a #path in the filesystem
+  # * Scan that path for subdirectories
+  # * Wrap each subdirectory with Hadupils::Extensions::Hive.
+  # * Aggregate their hivercs and their auxiliary jars
+  #
+  # See the Hadupils::Extensions::Hive class docs to understand
+  # the expectations per subdirectory.  The #path provided to
+  # HiveSet should be a directory that contains subdirectories conforming
+  # to Hadupils::Extensions::Hive conventions.
+  #
+  # All other files in the #path will be ignored; only subdirectories will
+  # be considered.
+  #
+  # == Member Extensions
+  #
+  # The Array of Hadupils::Extensions::Hive instances derived from
+  # #path's subdirectories will be available via the #members attribute
+  # reader.
+  #
+  # The order of members matches the lexicographic order of their
+  # respective subdirectory basenames within #path.
+  #
+  # The order of #hivercs and the order of #auxiliary_jars will follow
+  # the order of the respective #members.  All of member 0's #hivercs,
+  # followed by all of member 1's #hivercs, and so on.
+  #
+  # Thus the order of things is deterministic, according to lexicographic
+  # ordering of stuff in the filesystem.  You control it in how you
+  # lay stuff out.
+  #
+  # == Good Advice
+  #
+  # Don't do anything stupid.
+  #
+  class HiveSet
+    include Hive::AuxJarsPath
+
+    attr_reader :path
+    attr_reader :members
+
+    def initialize(path)
+      @path = ::File.expand_path(path)
+      @members = self.class.gather_member_extensions(@path)
+    end
+
+    def self.gather_member_extensions(path)
+      ::Dir.entries(path).sort.inject([]) do |result, entry|
+        full_path = ::File.join(path, entry)
+        if entry != '.' and entry != '..' and ::File.directory?(full_path)
+          result << Hive.new(full_path)
+        else
+          result
+        end
+      end
+    end
+
+    # The cumulative Array of hive initialization file objects
+    # across all #members, in member order.
+    def hivercs
+      members_inject {|member| member.hivercs}
+    end
+
+    # The cumulative Array of #auxiliary_jars across all #members,
+    # in member order.
+    def auxiliary_jars
+      members_inject {|member| member.auxiliary_jars}
+    end
+
+    # Accumulate a list based on an operation (given in a block)
+    # per member.  Accumulation is done against a starting empty list
+    # with the addition operator, not through appending.  Therefore,
+    # the block needs to provide an array, not an arbitrary object.
+    def members_inject
+      @members.inject [] do |result, member|
+        increment = yield member
+        result + increment
+      end
+    end
+  end
+end

data/lib/hadupils/extensions.rb

@@ -160,3 +160,5 @@ module Hadupils::Extensions
     end
   end
 end
+
+require 'hadupils/extensions/hive'

data/lib/hadupils/runners.rb

@@ -8,8 +8,32 @@ module Hadupils::Runners
 
     def command; end
 
+    def execute!
+      command_list = command
+      if RUBY_VERSION < '1.9' and command_list[0].kind_of? Hash
+        deletes = []
+        overrides = {}
+        begin
+          command_list[0].each do |key, val|
+            if ::ENV.has_key? key
+              overrides[key] = ::ENV[key]
+            else
+              deletes << key
+            end
+            ::ENV[key] = val
+          end
+          Kernel.system(*command_list[1..-1])
+        ensure
+          overrides.each {|key, val| ::ENV[key] = val }
+          deletes.each {|key| ::ENV.delete key }
+        end
+      else
+        Kernel.system(*command_list)
+      end
+    end
+
     def wait!
-      @last_result = Kernel.system(*command)
+      @last_result = execute!
       @last_status = $?
       if @last_result.nil?
         255
@@ -18,12 +42,17 @@ module Hadupils::Runners
       end
     end
 
-    def self.run(params)
-      self.new(params).wait!
+    def self.run(*params)
+      self.new(*params).wait!
     end
   end
 
   class Hive < Base
+    def initialize(params, hive_aux_jars_path='')
+      super(params)
+      @hive_aux_jars_path = hive_aux_jars_path
+    end
+
     def self.base_runner
       @base_runner || ::File.join(ENV['HIVE_HOME'], 'bin', 'hive')
     end
@@ -33,7 +62,7 @@ module Hadupils::Runners
     end
 
     def command
-      items = params.inject([self.class.base_runner]) do |result, param|
+      params.inject([env_overrides, self.class.base_runner]) do |result, param|
         if param.respond_to? :hive_opts
           result + param.hive_opts
         else
@@ -41,5 +70,16 @@ module Hadupils::Runners
         end
       end
     end
+
+    def env_overrides
+      e = {}
+      settings = [@hive_aux_jars_path, ::ENV['HIVE_AUX_JARS_PATH']].reject do |val|
+        val.nil? or val.strip == ''
+      end
+      if settings.length > 0
+        e['HIVE_AUX_JARS_PATH'] = settings.join(',')
+      end
+      e
+    end
   end
 end

data/lib/hadupils/search.rb

@@ -46,4 +46,22 @@ module Hadupils::Search
   def self.hadoop_assets
     find_from_pwd(hadoop_assets_name)
   end
+
+  # The basename to use when looking for hive extensions from pwd.
+  def self.hive_extensions_name
+    @hive_extensions_name || 'hive-ext'
+  end
+
+  # Set the basename to use when looking for hive assets from pwd.
+  def self.hive_extensions_name=(basename)
+    @hive_extensions_name = basename
+  end
+
+  # A search for #hive_extensions_name from the pwd.
+  # The default behavior is to look for a subdir named +hive-ext+,
+  # starting from the current working directory and walking upwards until
+  # a match is found or the file system root is encountered.
+  def self.hive_extensions
+    find_from_pwd(hive_extensions_name)
+  end
 end

data/test/hadupil_test_setup.rb

@@ -47,4 +47,16 @@ class Test::Unit::TestCase
       instance_eval &block
     end
   end
+
+  # Lets us define shared bits of shoulda context (setup blocks, tests,
+  # subcontexts, etc.) in a declarative manner; installs a singleton method
+  # :name into the calling class, that when invoked will eval the given
+  # block in the current Shoulda::Context::Context.
+  # You can then simply call :name in any arbitrary context in order to
+  # make use of the shared stuff within that context.
+  def self.shared_context(name, &block)
+    define_singleton_method name do
+      Shoulda::Context.current_context.instance_eval &block
+    end
+  end
 end

data/test/unit/commands_test.rb

@@ -63,28 +63,47 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
         cmd.user_config
       end
 
+      should 'have a HiveSet extension based on search for hive-ext' do
+        Hadupils::Search.expects(:hive_extensions).with.returns(path = mock())
+        Hadupils::Extensions::HiveSet.expects(:new).with(path).returns(extension = mock)
+        cmd = @klass.new
+        assert_equal extension, cmd.hive_ext
+        # Fails on expectations if previous result wasn't cached.
+        cmd.hive_ext
+      end
+
       context '#run' do
         setup do
           @command = @klass.new
           @command.stubs(:user_config).with.returns(@user_config = mock())
           @command.stubs(:hadoop_ext).with.returns(@hadoop_ext = mock())
+          @command.stubs(:hive_ext).with.returns(@hive_ext = mock)
           @runner_class = Hadupils::Runners::Hive
         end
 
-        context 'with user config and hadoop asssets hivercs' do
+        context 'with user config, hadoop assets, hive ext hivercs and aux jars' do
           setup do
             @user_config.stubs(:hivercs).returns(@user_config_hivercs = [mock(), mock()])
             @hadoop_ext.stubs(:hivercs).returns(@hadoop_ext_hivercs = [mock(), mock(), mock()])
+            @hive_ext.stubs(:hivercs).returns(@hive_ext_hivercs = [mock, mock, mock])
+            @hive_ext.stubs(:hive_aux_jars_path).returns(@hive_aux_jars_path = mock.to_s)
           end
 
           should 'apply hiverc options to hive runner call' do
-            @runner_class.expects(:run).with(@user_config_hivercs + @hadoop_ext_hivercs).returns(result = mock())
+            @runner_class.expects(:run).with(@user_config_hivercs +
+                                             @hadoop_ext_hivercs +
+                                             @hive_ext_hivercs,
+                                             @hive_aux_jars_path).returns(result = mock())
             assert_equal result, @command.run([])
           end
 
           should 'prepend hiverc options before given params to hive runner call' do
             params = [mock(), mock()]
-            @runner_class.expects(:run).with(@user_config_hivercs + @hadoop_ext_hivercs + params).returns(result = mock())
+            @runner_class.expects(:run).with(@user_config_hivercs +
+                                             @hadoop_ext_hivercs +
+                                             @hive_ext_hivercs +
+                                             params,
+                                             @hive_aux_jars_path).returns(result = mock())
             assert_equal result, @command.run(params)
           end
         end
@@ -93,15 +112,17 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
           setup do
             @user_config.stubs(:hivercs).returns([])
             @hadoop_ext.stubs(:hivercs).returns([])
+            @hive_ext.stubs(:hivercs).returns([])
+            @hive_ext.stubs(:hive_aux_jars_path).returns('')
           end
 
-          should 'pass params unchanged through to hive runner call' do
-            @runner_class.expects(:run).with(params = [mock(), mock()]).returns(result = mock())
+          should 'pass params unchanged through to hive runner call along with aux jars path' do
+            @runner_class.expects(:run).with(params = [mock(), mock()], '').returns(result = mock())
             assert_equal result, @command.run(params)
           end
 
           should 'handle empty params' do
-            @runner_class.expects(:run).with([]).returns(result = mock())
+            @runner_class.expects(:run).with([], '').returns(result = mock())
             assert_equal result, @command.run([])
           end
         end
@@ -111,8 +132,11 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
         setup do
           @conf = ::File.join(@tempdir.path, 'conf')
           @ext  = ::File.join(@tempdir.path, 'hadoop-ext')
+          @hive_ext = @tempdir.full_path('hive-ext')
+
           ::Dir.mkdir(@conf)
           ::Dir.mkdir(@ext)
+          ::Dir.mkdir(@hive_ext)
           @hiverc = @tempdir.file(File.join('conf', 'hiverc')) do |f|
             f.write(@static_hiverc_content = 'my static content;')
             f.path
@@ -124,21 +148,81 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
           @dynamic_hiverc_content = ["ADD FILE #{@ext_file}",
                                      "ADD JAR #{@ext_jar}",
                                      "ADD ARCHIVE #{@ext_tar}"].join(";\n") + ";\n"
+
+          # Assemble two entries under hive-ext
+          @hive_exts = %w(one two).inject({}) do |result, name|
+            state = result[name.to_sym] = {}
+            state[:path] = ::File.join(@hive_ext, name)
+
+            ::Dir.mkdir(state[:path])
+            state[:static_hiverc] = ::File.open(::File.join(state[:path], 'hiverc'), 'w') do |file|
+              file.write(state[:static_hiverc_content] = "#{name} static content")
+              file.path
+            end
+
+            assets = state[:assets] = %w(a.tar.gz b.txt c.jar).collect do |base|
+              ::File.open(::File.join(state[:path], "#{name}-#{base}"), 'w') do |file|
+                file.path
+              end
+            end
+
+            state[:dynamic_hiverc_content] = ["ADD ARCHIVE #{assets[0]};",
+                                              "ADD FILE #{assets[1]};",
+                                              "ADD JAR #{assets[2]};"].join("\n") + "\n"
+
+            aux_path = state[:aux_path] = ::File.join(state[:path], 'aux-jars')
+            ::Dir.mkdir(aux_path)
+            state[:aux_jars] = %w(boo foo).collect do |base|
+              ::File.open(::File.join(aux_path, "#{name}-#{base}.jar"), 'w') do |file|
+                file.path
+              end
+            end
+
+            state[:hive_aux_jars_path] = state[:aux_jars].join(',')
+
+            result
+          end
+
+          # Can't use a simple stub for this because other things are
+          # checked within ENV.  Use a teardown to reset to its original state.
+          @orig_hive_aux_jars_path = ENV['HIVE_AUX_JARS_PATH']
+          ::ENV['HIVE_AUX_JARS_PATH'] = env_aux = mock.to_s
+          @hive_aux_jars_path_val = [@hive_exts[:one][:hive_aux_jars_path],
+                                     @hive_exts[:two][:hive_aux_jars_path],
+                                     env_aux].join(',')
+
           @pwd       = ::Dir.pwd
           Hadupils::Search.stubs(:user_config).with.returns(@conf)
           Hadupils::Runners::Hive.stubs(:base_runner).with.returns(@hive_prog = '/opt/hive/bin/hive')
           ::Dir.chdir @tempdir.path
         end
 
+        teardown do
+          if @orig_hive_aux_jars_path
+            ENV['HIVE_AUX_JARS_PATH'] = @orig_hive_aux_jars_path
+          else
+            ENV.delete 'HIVE_AUX_JARS_PATH'
+          end
+        end
+
         should 'produce a valid set of parameters and hivercs' do
           Kernel.stubs(:system).with() do |*args|
-            args[0] == @hive_prog and
-            args[1] == '-i' and
-            File.open(args[2], 'r').read == @static_hiverc_content and
-            args[3] == '-i' and
-            File.open(args[4], 'r').read == @dynamic_hiverc_content and
-            args[5] == '--hiveconf' and
-            args[6] == 'my.foo=your.fu'
+            args[0] == {'HIVE_AUX_JARS_PATH' => @hive_aux_jars_path_val} and
+            args[1] == @hive_prog and
+            args[2] == '-i' and
+            File.open(args[3], 'r').read == @static_hiverc_content and
+            args[4] == '-i' and
+            File.open(args[5], 'r').read == @dynamic_hiverc_content and
+            args[6] == '-i' and
+            File.open(args[7], 'r').read == @hive_exts[:one][:dynamic_hiverc_content] and
+            args[8] == '-i' and
+            File.open(args[9], 'r').read == @hive_exts[:one][:static_hiverc_content] and
+            args[10] == '-i' and
+            File.open(args[11], 'r').read == @hive_exts[:two][:dynamic_hiverc_content] and
+            args[12] == '-i' and
+            File.open(args[13], 'r').read == @hive_exts[:two][:static_hiverc_content] and
+            args[14] == '--hiveconf' and
+            args[15] == 'my.foo=your.fu'
           end
           Hadupils::Commands.run 'hive', ['--hiveconf', 'my.foo=your.fu']
         end

data/test/unit/extensions/hive_test.rb

@@ -0,0 +1,257 @@
+class Hadupils::Extensions::HiveTest < Test::Unit::TestCase
+  shared_context :provide_hive_ext do
+    setup do
+      @ext = Hadupils::Extensions::Hive.new(@tempdir.path)
+    end
+  end
+
+  shared_context :hive_aux_jars_path_cases do
+    should 'assemble hive_aux_jars_path from auxiliary_jars appropriately for use with HIVE_AUX_JARS_PATH env' do
+      jars = [mock, mock, mock, mock].collect {|m| m.to_s}
+      @ext.expects(:auxiliary_jars).with.returns(jars.collect {|j| "  #{j}  "})
+      # Verifying whitespace trimming, as part of it.
+      assert_equal jars.join(','), @ext.hive_aux_jars_path
+    end
+  end
+
+  shared_context :empty_hiverc_cases do
+    should 'have an empty hivercs list' do
+      assert_equal [], @ext.hivercs
+    end
+  end
+
+  shared_context :empty_auxiliary_cases do
+    should 'have an empty auxiliary jars list' do
+      assert_equal [], @ext.auxiliary_jars
+    end
+  end
+
+  shared_context :valid_auxiliary_cases do
+    should 'provide jars within aux-jars as the auxiliary_jars list' do
+      assert_equal @aux_jars, @ext.auxiliary_jars
+    end
+  end
+
+  shared_context :static_hiverc_cases do
+    should 'have the hiverc path for the final entry in hivercs' do
+      assert_equal @hiverc_file, @ext.hivercs[-1].hive_opts[1]
+    end
+
+    should 'have a static hiverc resource' do
+      assert_equal '-i', @ext.hivercs[-1].hive_opts[0]
+    end
+  end
+
+  shared_context :single_hiverc_cases do
+    should 'have only a single hiverc in the hivercs list' do
+      assert_equal 1, @ext.hivercs.length
+    end
+  end
+
+  shared_context :dynamic_hiverc_cases do
+    should 'have a hiverc with appropriate asset-oriented commands as the first entry in the hivercs' do
+      File.open(@ext.hivercs[0].hive_opts[1], 'r') do |f|
+        assert_equal @asset_commands, f.read
+      end
+    end
+
+    should 'have a dynamic hiverc resource' do
+      assert_equal '-i', @ext.hivercs[0].hive_opts[0]
+    end
+  end
+
+  shared_context :has_auxiliary_path do
+    setup do
+      @aux = @tempdir.full_path('aux-jars')
+      ::Dir.mkdir(@aux)
+    end
+  end
+
+  shared_context :has_auxiliary_jars do
+    setup do
+      if @aux.nil?
+        @aux = @tempdir.full_path('aux-jars')
+        ::Dir.mkdir(@aux)
+      end
+      @aux_jars = %w(a b c).collect do |base|
+        f = @tempdir.file(::File.join('aux-jars', "#{base}.jar"))
+        f.close
+        f.path
+      end
+      %w(tar.gz txt yaml).each do |extension|
+        @tempdir.file(::File.join('aux-jars', "bogus.#{extension}"))
+      end
+    end
+  end
+
+  shared_context :has_hiverc_file do
+    setup do
+      f = @tempdir.file('hiverc')
+      @hiverc_file = f.path
+      f.close
+    end
+  end
+
+  shared_context :has_assets do
+    setup do
+      @assets = %w{a.archive.tar.gz a.file.txt a.jar}.collect do |asset|
+        f = @tempdir.file(asset)
+        f.close
+        f.path
+      end
+
+      @asset_commands = "ADD ARCHIVE #{@assets[0]};\n" +
+                        "ADD FILE #{@assets[1]};\n" +
+                        "ADD JAR #{@assets[2]};\n"
+    end
+  end
+
+  tempdir_context Hadupils::Extensions::Hive do
+    context 'with auxiliary jars' do
+      provide_hive_ext
+      hive_aux_jars_path_cases
+    end
+
+    context 'given an empty directory' do
+      provide_hive_ext
+      empty_hiverc_cases
+      empty_auxiliary_cases
+    end
+
+    context 'given an empty aux-jars directory' do
+      has_auxiliary_path
+      provide_hive_ext
+      empty_hiverc_cases
+      empty_auxiliary_cases
+    end
+
+    context 'given a hiverc file' do
+      has_hiverc_file
+      provide_hive_ext
+      empty_auxiliary_cases
+      static_hiverc_cases
+      single_hiverc_cases
+    end
+
+    context 'given assets' do
+      has_assets
+
+      context 'and nothing else' do
+        provide_hive_ext
+        empty_auxiliary_cases
+        dynamic_hiverc_cases
+        single_hiverc_cases
+      end
+
+      context 'and a hiverc file' do
+        has_hiverc_file
+        provide_hive_ext
+        empty_auxiliary_cases
+        dynamic_hiverc_cases
+        static_hiverc_cases
+      end
+    end
+
+    context 'given a directory with an aux-jars directory and jars' do
+      has_auxiliary_jars
+
+      context 'and nothing else' do
+        provide_hive_ext
+        empty_hiverc_cases
+        valid_auxiliary_cases
+      end
+
+      context 'and assets' do
+        has_assets
+        provide_hive_ext
+        valid_auxiliary_cases
+        dynamic_hiverc_cases
+        single_hiverc_cases
+      end
+
+      context 'and a hiverc file' do
+        has_hiverc_file
+
+        context 'and no assets' do
+          provide_hive_ext
+          static_hiverc_cases
+          valid_auxiliary_cases
+          single_hiverc_cases
+        end
+
+        context 'and assets' do
+          has_assets
+          provide_hive_ext
+          dynamic_hiverc_cases
+          static_hiverc_cases
+          valid_auxiliary_cases
+        end
+      end
+    end
+  end
+
+  tempdir_context Hadupils::Extensions::HiveSet do
+    setup do
+      @cls = Hadupils::Extensions::HiveSet
+    end
+
+    should 'have the dir path expanded as :path' do
+      # Making the path relative demonstrates path expansion
+      ::Dir.chdir(::File.dirname(@tempdir.path)) do
+        assert_equal @tempdir.path,
+                     @cls.new(::File.basename(@tempdir.path)).path
+      end
+    end
+
+    should 'produce a Hadupils::Extensions::Hive per subdirectory' do
+      ::Dir.mkdir(a = @tempdir.full_path('aye'))
+      ::Dir.mkdir(b = @tempdir.full_path('bee'))
+      ::Dir.mkdir(c = @tempdir.full_path('si'))
+
+      # These should be ignored 'cause they ain't dirs
+      @tempdir.file('foo.txt')
+      @tempdir.file('blah.jar')
+      @tempdir.file('garbage.tar.gz')
+
+      expect = [a, b, c].collect {|path| [Hadupils::Extensions::Hive, path]}
+      ext = @cls.new(@tempdir.path)
+      assert_equal expect,
+                   ext.members.collect {|member| [member.class, member.path]}
+    end
+
+    context 'with members' do
+      setup do
+        @member_a = mock
+        @member_b = mock
+        @cls.expects(:gather_member_extensions).with(@tempdir.path).returns(@members = [@member_a, @member_b])
+        @ext = @cls.new @tempdir.path
+      end
+
+      hive_aux_jars_path_cases
+
+      should 'base members list on :gather_member_extensions' do
+        assert_equal @members, @ext.members
+      end
+
+      should 'produce the cumulative hivercs list from its members' do
+        hivercs = @members.inject([]) do |expect, member|
+          this_pair = [mock, mock]
+          member.expects(:hivercs).with.returns(this_pair)
+          expect + this_pair
+        end
+
+        assert_equal hivercs, @ext.hivercs
+      end
+
+      should 'produce the cumulative auxiliary_jars list from its members' do
+        jarries = @members.inject([]) do |expect, member|
+          jars = [mock, mock, mock].collect {|jar| jar.to_s}
+          member.expects(:auxiliary_jars).with.returns(jars)
+          expect + jars
+        end
+
+        assert_equal jarries, @ext.auxiliary_jars
+      end
+    end
+  end
+end

data/test/unit/runners_test.rb

@@ -11,26 +11,76 @@ class Hadupils::RunnersTest < Test::Unit::TestCase
     context 'wait!' do
       setup do
         @command = [mock(), mock(), mock()]
-        @runner.expects(:command).with.returns(@command)
         # This will ensure that $? is non-nil
         system(RbConfig.ruby, '-v')
       end
 
-      should 'assemble system call via command method' do
-        Kernel.expects(:system).with(*@command).returns(true)
-        $?.stubs(:exitstatus).with.returns(mock())
-        @runner.wait!
-      end
+      context 'with semi-modern ruby' do
+        setup do
+          @runner.expects(:command).with.returns(@command)
+        end
+
+        should 'assemble system call via command method' do
+          Kernel.expects(:system).with(*@command).returns(true)
+          $?.stubs(:exitstatus).with.returns(mock())
+          @runner.wait!
+        end
 
-      should 'return 255 when system returns nil' do
-        Kernel.stubs(:system).returns(nil)
-        assert_equal 255, @runner.wait!
+        should 'return 255 when system returns nil' do
+          Kernel.stubs(:system).returns(nil)
+          assert_equal 255, @runner.wait!
+        end
+
+        should 'return Process::Status#exitstatus when non-nil system result' do
+          Kernel.stubs(:system).returns(true)
+          $?.stubs(:exitstatus).with.returns(status = mock())
+          assert_equal status, @runner.wait!
+        end
       end
 
-      should 'return Process::Status#exitstatus when non-nil system result' do
-        Kernel.stubs(:system).returns(true)
-        $?.stubs(:exitstatus).with.returns(status = mock())
-        assert_equal status, @runner.wait!
+      context 'with ruby pre 1.9' do
+        setup do
+          @orig_ruby_version = ::RUBY_VERSION
+          ::RUBY_VERSION = '1.8.7'
+        end
+
+        teardown do
+          ::RUBY_VERSION = @orig_ruby_version
+        end
+
+        should 'handle command without env hash normally' do
+          @runner.expects(:command).with.returns(@command)
+          Kernel.expects(:system).with(*@command).returns(true)
+          $?.stubs(:exitstatus).with.returns(mock)
+          @runner.wait!
+        end
+
+        should 'handle environment hash specially and restore env' do
+          # A defined environment variable to play with.
+          var = ::ENV.keys.find {|k| ENV[k].strip.length > 0}
+          orig = ::ENV[var]
+          to_be_removed = ::ENV.keys.sort[-1] + 'X'
+          removal_val = mock.to_s
+          replacement = "#{orig}-#{mock.to_s}"
+          @runner.expects(:command).with.returns([{var => replacement, to_be_removed => removal_val}] + @command)
+          $?.stubs(:exitstatus).with.returns(mock)
+          begin
+            # Environment variable is overridden during system call
+            matcher = Kernel.expects(:system).with do |*args|
+              args == @command and ::ENV[var] == replacement and ::ENV[to_be_removed] == removal_val
+            end
+
+            matcher.returns true
+
+            @runner.wait!
+
+            # But is restored afterward
+            assert_equal orig, ::ENV[var]
+            assert_equal false, ::ENV.has_key?(to_be_removed)
+          ensure
+            ::ENV[var] = orig
+          end
+        end
       end
     end
   end
@@ -56,12 +106,25 @@ class Hadupils::RunnersTest < Test::Unit::TestCase
       end
 
       should 'provide invocation for bare hive if given empty parameters' do
-        assert_equal [@hive_path], @klass.new([]).command
+        assert_equal [{}, @hive_path], @klass.new([]).command
+      end
+
+      should 'provide invocation with aux jars and bare hive given empty params but aux jars path' do
+        ENV.stubs(:[]=).with('HIVE_AUX_JARS_PATH').returns(nil)
+        assert_equal [{'HIVE_AUX_JARS_PATH' => 'foo'}, @hive_path],
+                     @klass.new([], 'foo').command
+      end
+
+      should 'provide invocation with merged aux jars given otherwise bare stuff' do
+        ::ENV.stubs(:[]).with('HIVE_AUX_JARS_PATH').returns(orig = mock.to_s)
+        additional = mock.to_s
+        assert_equal [{'HIVE_AUX_JARS_PATH' => "#{additional},#{orig}"}, @hive_path],
+                     @klass.new([], additional).command
       end
 
       should 'provide invocation for hive with all given parameters' do
         params = [mock().to_s, mock().to_s, mock().to_s, mock().to_s]
-        assert_equal [@hive_path] + params,
+        assert_equal [{}, @hive_path] + params,
                      @klass.new(params).command
       end
 
@@ -72,7 +135,7 @@ class Hadupils::RunnersTest < Test::Unit::TestCase
         p2.expects(:hive_opts).with.returns(p2_opts = ['-i', mock().to_s])
         s1 = mock().to_s
         s2 = mock().to_s
-        assert_equal [@hive_path, s1] + p1_opts + [s2] + p2_opts,
+        assert_equal [{}, @hive_path, s1] + p1_opts + [s2] + p2_opts,
                      @klass.new([s1, p1, s2, p2]).command
       end
     end

data/test/unit/search_test.rb

@@ -80,4 +80,22 @@ class Hadupils::SearchTest < Test::Unit::TestCase
       assert_respond_to Hadupils::Search, :hadoop_assets_name=
     end
   end
+
+  context 'hive_extensions' do
+    should 'search for a directory specified by #hive_extensions_name' do
+      Hadupils::Search.expects(:hive_extensions_name).with.returns(name = mock.to_s)
+      Hadupils::Search.expects(:find_from_pwd).with(name).returns(dir = mock)
+      assert_equal dir, Hadupils::Search.hive_extensions
+    end
+  end
+
+  context 'hive_extensions_name' do
+    should 'default to "hive-ext"' do
+      assert_equal 'hive-ext', Hadupils::Search.hive_extensions_name
+    end
+
+    should 'be settable' do
+      assert_respond_to Hadupils::Search, :hive_extensions_name=
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hadupils
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -85,14 +85,16 @@ files:
 - lib/hadupils/search.rb
 - lib/hadupils/commands.rb
 - lib/hadupils/runners.rb
+- lib/hadupils/extensions/hive.rb
 - lib/hadupils/extensions.rb
 - lib/hadupils/assets.rb
 - lib/hadupils.rb
 - test/unit/assets_test.rb
 - test/unit/commands_test.rb
-- test/unit/extensions_test.rb
 - test/unit/runners_test.rb
 - test/unit/search_test.rb
+- test/unit/extensions/base_test.rb
+- test/unit/extensions/hive_test.rb
 - test/hadupil_test_setup.rb
 - bin/hadupils
 - Rakefile.rb