hadupils 0.1.1

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+
+### 0.1.0
+
+* Basic functionality for representing a command, extensions,
+  assets, runners.
+* A hive runner enforcing user config and hadoop-ext extension
+* A hadupils executable for entering the command model
+
+### 0.1.1
+
+* Removed evil rubygems requirement from executable
+* Added this glorious changelog
+

data/Gemfile

@@ -0,0 +1,6 @@
+source :rubygems
+
+gem 'mocha',            :group => :test
+gem 'rake', '10.1.0',   :group => :development
+gem 'shoulda-context',  :group => :test
+

data/Gemfile.lock

@@ -0,0 +1,16 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    metaclass (0.0.1)
+    mocha (0.14.0)
+      metaclass (~> 0.0.1)
+    rake (10.1.0)
+    shoulda-context (1.1.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mocha
+  rake (= 10.1.0)
+  shoulda-context

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Ethan Rowe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+hadupils
+========
+
+Operating environment oriented utilities for hadoop (Hadoop + Utils => hadupils)

data/Rakefile.rb

@@ -0,0 +1,13 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+
+CLOBBER.include('hadupils-*.gem')
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/**/*_test.rb'
+  t.libs = ['test', 'lib']
+  # This should initialize the environment properly.
+  t.ruby_opts << '-rhadupil_test_setup'
+end
+

data/bin/hadupils

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+# vim: set filetype=ruby:
+
+require 'hadupils'
+
+exit Hadupils::Commands.run ARGV[0], ARGV[1..-1]
+

data/lib/hadupils/assets.rb

@@ -0,0 +1,57 @@
+module Hadupils::Assets
+  class File
+    attr_reader :name, :path
+
+    def self.determine_name(path)
+      ::File.basename(path)
+    end
+
+    def self.hiverc_type
+      :FILE
+    end
+
+    def initialize(path)
+      @path = path
+      @name = self.class.determine_name(path)
+    end
+
+    def hidden?
+      name[0] == '.'
+    end
+
+    def hiverc_command
+      "ADD #{self.class.hiverc_type} #{path};"
+    end
+  end
+
+  class Jar < File
+    def self.hiverc_type
+      :JAR
+    end
+  end
+
+  class Archive < File
+    def self.hiverc_type
+      :ARCHIVE
+    end
+  end
+
+  def self.asset_for(path)
+    return Archive.new(path) if path[-7..-1] == '.tar.gz'
+    return Jar.new(path) if path[-4..-1] == '.jar'
+    return File.new(path)
+  end
+
+  SKIP_NAMES = ['.', '..']
+
+  # Walks the top-level members of the stated directory and
+  # returns an array containing appropriate an HadoopAsset::*
+  # instance for each.
+  def self.assets_in(directory)
+    path = ::File.expand_path(directory)
+    ::Dir.entries(path).sort.inject([]) do |accum, entry|
+      accum << asset_for(::File.join(path, entry)) if not SKIP_NAMES.include? entry
+      accum
+    end
+  end
+end

data/lib/hadupils/commands.rb

@@ -0,0 +1,48 @@
+module Hadupils::Commands
+  def self.run(command, params)
+    handler = handler_for command
+    handler.run params
+  end
+
+  def self.handler_for(command)
+    @handlers and @handlers[command.downcase.to_sym]
+  end
+
+  def self.register_handler(command, handler)
+    @handlers ||= {}
+    @handlers[command.downcase.to_sym] = handler
+  end
+
+  class SimpleCommand
+    def self.run(params)
+      self.new.run params
+    end
+  end
+
+  module HadoopExt
+    def hadoop_ext
+      @hadoop_ext ||= Hadupils::Extensions::Flat.new(Hadupils::Search.hadoop_assets)
+    end
+  end
+
+  module UserConf
+    def user_config
+      @user_config ||= Hadupils::Extensions::Static.new(Hadupils::Search.user_config)
+    end
+  end
+
+  class Hive < SimpleCommand
+    include HadoopExt
+    include UserConf
+
+    def assemble_parameters(parameters)
+      user_config.hivercs + hadoop_ext.hivercs + parameters
+    end
+
+    def run(parameters)
+      Hadupils::Runners::Hive.run assemble_parameters(parameters)
+    end
+  end
+
+  register_handler :hive, Hive
+end

data/lib/hadupils/extensions.rb

@@ -0,0 +1,162 @@
+module Hadupils::Extensions
+  # Tools for managing hive initialization files ("hiverc").
+  module HiveRC
+    module HiveOpt
+      def hive_opts
+        ['-i', path]
+      end
+    end
+
+    # Wraps an extant hive initialization file, and providing
+    # an interface compatible with the critical parts of the
+    # Dynamic sibling class so they may be used interchangeably
+    # by runners when determining hive options.
+    class Static
+      attr_reader :path
+
+      include HiveOpt
+
+      # Given a path, expands it ti
+      def initialize(path)
+        @path = ::File.expand_path(path)
+      end
+
+      def close
+      end
+    end
+
+    # Manages dynamic hive initialization files, assembling a temporary file
+    # and understanding how to write assets/lines into the initialization file for
+    # use with hive's -i option.
+    class Dynamic
+      attr_reader :file
+
+      include HiveOpt
+      require 'tempfile'
+
+      # This will allow us to change what handles the dynamic files.
+      def self.file_handler=(handler)
+        @file_handler = handler
+      end
+
+      # The class to use for creating the files; defaults to ::Tempfile
+      def self.file_handler
+        @file_handler || ::Tempfile
+      end
+
+      # Sets up a wrapped file, using the class' file_handler, 
+      def initialize
+        @file = self.class.file_handler.new('hadupils-hiverc')
+      end
+
+      def path
+        ::File.expand_path @file.path
+      end
+
+      def close
+        @file.close
+      end
+
+      # Writes the items to the file, using #hiverc_command on each item that
+      # responds to it (Hadupils::Assets::* instances) and #to_s on the rest.
+      # Separates lines by newline, and provides a trailing newline.  However,
+      # the items are responsible for ensuring the proper terminating semicolon.
+      # The writes are flushed to the underlying file immediately afterward.
+      def write(items)
+        lines = items.collect do |item|
+          if item.respond_to? :hiverc_command
+            item.hiverc_command
+          else
+            item.to_s
+          end
+        end
+        @file.write(lines.join("\n") + "\n")
+        @file.flush
+      end
+    end
+  end
+
+  class EvalProxy
+    def initialize(scope)
+      @scope = scope
+    end
+
+    def assets(&block)
+      @scope.instance_eval do
+        @assets_block = block
+      end
+    end
+
+    def hiverc(&block)
+      @scope.instance_eval do
+        @hiverc_block = block
+      end
+    end
+  end
+
+  class Base
+    attr_reader :assets, :path
+
+    def initialize(directory, &block)
+      if block_given?
+        EvalProxy.new(self).instance_eval &block
+      end
+      @path = ::File.expand_path(directory) unless directory.nil?
+      @assets = merge_assets(self.class.gather_assets(@path))
+    end
+
+    def merge_assets(assets)
+      return @assets_block.call(assets) if @assets_block
+      assets
+    end
+
+    def hivercs
+      []
+    end
+
+    def self.gather_assets(directory)
+      if not directory.nil?
+        Hadupils::Assets.assets_in(directory)
+      else
+        []
+      end
+    end
+  end
+
+  class Flat < Base
+    def hivercs
+      @hiverc ||= assemble_hiverc
+      [@hiverc]
+    end
+
+    def assemble_hiverc
+      assets = @assets
+      if @hiverc_block
+        assets = @hiverc_block.call(assets.dup)
+      end
+      hiverc = Hadupils::Extensions::HiveRC::Dynamic.new
+      hiverc.write(assets)
+      hiverc
+    end
+  end
+
+  class Static < Base
+    def self.gather_assets(path)
+      []
+    end
+
+    def hiverc_path
+      ::File.join(path, 'hiverc')
+    end
+
+    def hiverc?
+      ::File.file? hiverc_path
+    end
+
+    def hivercs
+      r = []
+      r << Hadupils::Extensions::HiveRC::Static.new(hiverc_path) if hiverc?
+      r
+    end
+  end
+end

data/lib/hadupils/runners.rb

@@ -0,0 +1,45 @@
+module Hadupils::Runners
+  class Base
+    attr_reader :params, :last_result, :last_status
+
+    def initialize(params)
+      @params = params
+    end
+
+    def command; end
+
+    def wait!
+      @last_result = Kernel.system(*command)
+      @last_status = $?
+      if @last_result.nil?
+        255
+      else
+        @last_status.exitstatus
+      end
+    end
+
+    def self.run(params)
+      self.new(params).wait!
+    end
+  end
+
+  class Hive < Base
+    def self.base_runner
+      @base_runner || ::File.join(ENV['HIVE_HOME'], 'bin', 'hive')
+    end
+
+    def self.base_runner=(runner_path)
+      @base_runner = runner_path
+    end
+
+    def command
+      items = params.inject([self.class.base_runner]) do |result, param|
+        if param.respond_to? :hive_opts
+          result + param.hive_opts
+        else
+          result << param
+        end
+      end
+    end
+  end
+end

data/lib/hadupils/search.rb

@@ -0,0 +1,49 @@
+module Hadupils::Search
+  # Searches for directory containing a subdirectory `name`, starting at
+  # the specified `start` directory and walking upwards until it can go
+  # no farther.  On first match, the absolute path to that subdirectory
+  # is returned.  If not found, returns nil.
+  def self.find_from_dir(name, start)
+    curr = ::File.expand_path(start)
+    last = nil
+    while curr != last
+      p = ::File.join(curr, name)
+      return p if ::File.directory? p
+      last = curr
+      curr = ::File.dirname(curr)
+    end
+    nil
+  end
+
+  # Performs a `find_from_dir` starting at the present working directory.
+  def self.find_from_pwd(name)
+    find_from_dir(name, ::Dir.pwd)
+  end
+
+  # The directory for user-specific configuration files.
+  def self.user_config
+    @user_config || ::File.expand_path(::File.join('~', 'conf'))
+  end
+
+  def self.user_config=(path)
+    @user_config = path
+  end
+
+  # The basename to use when looking for hadoop assets from pwd.
+  def self.hadoop_assets_name
+    @hadoop_assets_name || 'hadoop-ext'
+  end
+
+  # Set the basename to use when looking for hadoop assets from pwd.
+  def self.hadoop_assets_name=(basename)
+    @hadoop_assets_name = basename
+  end
+
+  # A search for #hadoop_assets_name from the pwd.
+  # The default behavior is to look for a subdir named "hadoop-ext",
+  # starting from the current working directory and walking upwards until
+  # a match is found or the file system root is encountered.
+  def self.hadoop_assets
+    find_from_pwd(hadoop_assets_name)
+  end
+end

data/lib/hadupils.rb

@@ -0,0 +1,9 @@
+
+module Hadupils
+end
+
+require 'hadupils/assets'
+require 'hadupils/commands'
+require 'hadupils/extensions'
+require 'hadupils/runners'
+require 'hadupils/search'

data/test/hadupil_test_setup.rb

@@ -0,0 +1,50 @@
+require 'bundler'
+Bundler.setup
+require 'test/unit'
+require 'shoulda-context'
+require 'mocha/setup'
+require 'tempfile'
+require 'hadupils'
+
+# Add tempdir niceties to Test::Unit::TestCase
+# on top of the shoulda-context stuff.
+class Test::Unit::TestCase
+  class DirWrapper
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def full_path(path)
+      ::File.expand_path(::File.join(@path, path))
+    end
+
+    def file(path)
+      path = full_path(path)
+      if block_given?
+        ::File.open(path, 'w') do |f|
+          yield f
+        end
+      else
+        ::File.new(path, 'w')
+      end
+    end
+  end
+
+  def self.tempdir_context(name, &block)
+    context name do
+      setup do
+        @tempdir = Test::Unit::TestCase::DirWrapper.new(::File.expand_path(::Dir.mktmpdir))
+      end
+
+      teardown do
+        FileUtils.remove_entry @tempdir.path
+      end
+
+      # Instance_eval instead of simple yield to ensure it happens in the Context object
+      # and not in the test case subclass.
+      instance_eval &block
+    end
+  end
+end

data/test/unit/assets_test.rb

@@ -0,0 +1,166 @@
+class AssetsTest < Test::Unit::TestCase
+  context 'a file' do
+    setup do
+      @path = '/foo/bar/some_file.blah'
+      @asset = Hadupils::Assets::File.new(@path)
+    end
+
+    should 'have a path' do
+      assert_equal @path, @asset.path
+    end
+
+    should 'have a name' do
+      assert_equal ::File.basename(@path), @asset.name
+    end
+
+    should 'have an ADD FILE hiverc command' do
+      assert_equal "ADD FILE #{@path};", @asset.hiverc_command
+    end
+
+    should 'not be hidden' do
+      assert_equal false, @asset.hidden?
+    end
+  end
+
+  context 'a jar' do
+    setup do
+      @path = '/blah/blargh/../foo/blee/something.jar'
+      @asset = Hadupils::Assets::Jar.new(@path)
+    end
+
+    should 'have a path' do
+      assert_equal @path, @asset.path
+    end
+
+    should 'have a name' do
+      assert_equal ::File.basename(@path), @asset.name
+    end
+
+    should 'have an ADD JAR hiverc command' do
+      assert_equal "ADD JAR #{@path};", @asset.hiverc_command
+    end
+
+    should 'not be hidden' do
+      assert_equal false, @asset.hidden?
+    end
+  end
+
+  context 'a tarball' do
+    setup do
+      @path = '/blah/blargh/../foo/blee/something.tar.gz'
+      @asset = Hadupils::Assets::Archive.new(@path)
+    end
+
+    should 'have a path' do
+      assert_equal @path, @asset.path
+    end
+
+    should 'have a name' do
+      assert_equal ::File.basename(@path), @asset.name
+    end
+
+    should 'have an ADD ARCHIVE hiverc command' do
+      assert_equal "ADD ARCHIVE #{@path};", @asset.hiverc_command
+    end
+
+    should 'not be hidden' do
+      assert_equal false, @asset.hidden?
+    end
+  end
+
+  context 'a hidden file' do
+    should 'have a hidden File asset' do
+      assert_equal true, Hadupils::Assets::File.new('/blah/blee/.foo.bar.txt').hidden?
+    end
+
+    should 'have a hidden Archive asset' do
+      assert_equal true, Hadupils::Assets::Archive.new('/floo/flam/.foo.tar.gz').hidden?
+    end
+
+    should 'have a hidden Jar asset' do
+      assert_equal true, Hadupils::Assets::Jar.new('/flibbidy/blop/.foo.bar.jar').hidden?
+    end
+  end
+
+  context 'asset_for' do
+    context 'given a file of no particular extension' do
+      setup do
+        @path = '/some/special/file.path'
+        @asset = Hadupils::Assets.asset_for(@path)
+      end
+
+      should 'produce a Hadupils::Assets::File' do
+        assert_same Hadupils::Assets::File, @asset.class
+      end
+
+      should 'pass the path through' do
+        assert_equal @path, @asset.path
+      end
+    end
+
+    context 'given a file with a .jar extension' do
+      setup do
+        @path = '/some/great/magical-1.7.9.jar'
+        @asset = Hadupils::Assets.asset_for(@path)
+      end
+
+      should 'product a Hadupils::Assets::Jar' do
+        assert_same Hadupils::Assets::Jar, @asset.class
+      end
+
+      should 'pass the path through' do
+        assert_equal @path, @asset.path
+      end
+    end
+
+    context 'given a file with a .tar.gz extension' do
+      setup do
+        @path = '/some/freaking/awesome.tar.gz'
+        @asset = Hadupils::Assets.asset_for(@path)
+      end
+
+      should 'produce a Hadupils::Assets::Archive' do
+        assert_same Hadupils::Assets::Archive, @asset.class
+      end
+
+      should 'pass the path through' do
+        assert_equal @path, @asset.path
+      end
+    end
+  end
+
+  tempdir_context 'a directory with files' do
+    setup do
+      @tempdir.file(@archive = 'Awesome-archive.tar.gz')
+      @tempdir.file(@jar = 'jarrIE.jar')
+      @tempdir.file(@file = 'sumyummy.yaml')
+      @matches = {@archive => Hadupils::Assets::Archive,
+                  @jar     => Hadupils::Assets::Jar,
+                  @file    => Hadupils::Assets::File}
+    end
+
+    context 'given to Hadupils::Assets.assets_in' do
+      setup do
+        @received = Hadupils::Assets.assets_in(@tempdir.path)
+      end
+
+      should 'get assets in lexicographic order' do
+        assert_equal @matches.keys.sort, (@received.collect {|a| a.name})
+      end
+
+      should 'get assets of appropriate type' do
+        type_map = @received.inject({}) {|hash, asset| hash[asset.name] = asset.class; hash}
+        assert_equal @matches, type_map
+      end
+
+      should 'get assets with expanded paths' do
+        path_map = @received.inject({}) {|hash, asset| hash[asset.name] = asset.path; hash}
+        expected = @matches.keys.inject({}) do |hash, name|
+          hash[name] = @tempdir.full_path(name)
+          hash
+        end
+        assert_equal expected, path_map
+      end
+    end
+  end
+end

data/test/unit/commands_test.rb

@@ -0,0 +1,135 @@
+class Hadupils::CommandsTest < Test::Unit::TestCase
+  context Hadupils::Commands do
+    context 'run singleton method' do
+      should 'pass trailing params to #run method of handler identified by first param' do
+        Hadupils::Commands.expects(:handler_for).with(cmd = mock()).returns(handler = mock())
+        handler.expects(:run).with(params = [mock(), mock(), mock()]).returns(result = mock())
+        assert_equal result, Hadupils::Commands.run(cmd, params)
+      end
+    end
+
+    context 'Hive' do
+      setup do
+        @klass = Hadupils::Commands::Hive
+      end
+
+      should 'register with :hive name' do
+        assert_same @klass, Hadupils::Commands.handler_for(:hive)
+        assert_same @klass, Hadupils::Commands.handler_for(:HivE)
+        assert_same @klass, Hadupils::Commands.handler_for('hive')
+        assert_same @klass, Hadupils::Commands.handler_for('hIVe')
+      end
+
+      should 'have a #run singleton method that dispatches to an instance #run' do
+        @klass.expects(:new).with.returns(instance = mock())
+        instance.expects(:run).with(params = mock()).returns(result = mock())
+        assert_equal result, @klass.run(params)
+      end
+
+      should 'have a Flat extension based on a search for hadoop-ext' do
+        Hadupils::Search.expects(:hadoop_assets).with.returns(assets = mock())
+        Hadupils::Extensions::Flat.expects(:new).with(assets).returns(extension = mock())
+        cmd = @klass.new
+        assert_equal extension, cmd.hadoop_ext
+        # This should cause failure if the previous result wasn't
+        # cached internally (by breaking expectations).
+        cmd.hadoop_ext
+      end
+
+      should 'have a Static extensions based on user config' do
+        Hadupils::Search.expects(:user_config).with.returns(conf = mock())
+        Hadupils::Extensions::Static.expects(:new).with(conf).returns(extension = mock())
+        cmd = @klass.new
+        assert_equal extension, cmd.user_config
+        # Fails on expectations if previous result wasn't cached.
+        cmd.user_config
+      end
+
+      context '#run' do
+        setup do
+          @command = @klass.new
+          @command.stubs(:user_config).with.returns(@user_config = mock())
+          @command.stubs(:hadoop_ext).with.returns(@hadoop_ext = mock())
+          @runner_class = Hadupils::Runners::Hive
+        end
+
+        context 'with user config and hadoop asssets hivercs' do
+          setup do
+            @user_config.stubs(:hivercs).returns(@user_config_hivercs = [mock(), mock()])
+            @hadoop_ext.stubs(:hivercs).returns(@hadoop_ext_hivercs = [mock(), mock(), mock()])
+          end
+
+          should 'apply hiverc options to hive runner call' do
+            @runner_class.expects(:run).with(@user_config_hivercs + @hadoop_ext_hivercs).returns(result = mock())
+            assert_equal result, @command.run([])
+          end
+
+          should 'prepend hiverc options before given params to hive runner call' do
+            params = [mock(), mock()]
+            @runner_class.expects(:run).with(@user_config_hivercs + @hadoop_ext_hivercs + params).returns(result = mock())
+            assert_equal result, @command.run(params)
+          end
+        end
+
+        context 'without hivercs' do
+          setup do
+            @user_config.stubs(:hivercs).returns([])
+            @hadoop_ext.stubs(:hivercs).returns([])
+          end
+
+          should 'pass params unchanged through to hive runner call' do
+            @runner_class.expects(:run).with(params = [mock(), mock()]).returns(result = mock())
+            assert_equal result, @command.run(params)
+          end
+
+          should 'handle empty params' do
+            @runner_class.expects(:run).with([]).returns(result = mock())
+            assert_equal result, @command.run([])
+          end
+        end
+      end
+
+      tempdir_context 'running for (mostly) realz' do
+        setup do
+          @conf = ::File.join(@tempdir.path, 'conf')
+          @ext  = ::File.join(@tempdir.path, 'hadoop-ext')
+          ::Dir.mkdir(@conf)
+          ::Dir.mkdir(@ext)
+          @hiverc = @tempdir.file(File.join('conf', 'hiverc')) do |f|
+            f.write(@static_hiverc_content = 'my static content;')
+            f.path
+          end
+          file = Proc.new {|base, name| @tempdir.file(::File.join(base, name)).path }
+          @ext_file  = file.call('hadoop-ext', 'a_file.yaml')
+          @ext_jar   = file.call('hadoop-ext', 'a_jar.jar')
+          @ext_tar   = file.call('hadoop-ext', 'a_tar.tar.gz')
+          @dynamic_hiverc_content = ["ADD FILE #{@ext_file}",
+                                     "ADD JAR #{@ext_jar}",
+                                     "ADD ARCHIVE #{@ext_tar}"].join(";\n") + ";\n"
+          @pwd       = ::Dir.pwd
+          Hadupils::Search.stubs(:user_config).with.returns(@conf)
+          Hadupils::Runners::Hive.stubs(:base_runner).with.returns(@hive_prog = '/opt/hive/bin/hive')
+          ::Dir.chdir @tempdir.path
+        end
+
+        should 'produce a valid set of parameters and hivercs' do
+          Kernel.stubs(:system).with() do |*args|
+            args[0] == @hive_prog and
+            args[1] == '-i' and
+            File.open(args[2], 'r').read == @static_hiverc_content and
+            args[3] == '-i' and
+            File.open(args[4], 'r').read == @dynamic_hiverc_content and
+            args[5] == '--hiveconf' and
+            args[6] == 'my.foo=your.fu'
+          end
+          Hadupils::Commands.run 'hive', ['--hiveconf', 'my.foo=your.fu']
+        end
+
+        teardown do
+          ::Dir.chdir @pwd
+        end
+      end
+    end
+  end
+end
+

data/test/unit/extensions_test.rb

@@ -0,0 +1,266 @@
+class Hadupils::ExtensionsTest < Test::Unit::TestCase
+  context Hadupils::Extensions::Base do
+    context 'initialization with nil path' do
+      should 'have nil as the path' do
+        ext = Hadupils::Extensions::Base.new(nil)
+        assert_equal nil, ext.path
+      end
+    end
+
+    context 'initialization' do
+      setup do
+        @path = mock()
+        @expanded_path = mock()
+        @assets = mock()
+        ::File.expects(:expand_path).with(@path).returns(@expanded_path)
+      end
+
+      should 'expand the given directory into :path' do
+        Hadupils::Extensions::Base.stubs(:gather_assets).returns(@assets)
+        assert_equal @expanded_path, Hadupils::Extensions::Base.new(@path).path
+      end
+
+      should "gather the expanded directory's assets" do
+        Hadupils::Extensions::Base.expects(:gather_assets).with(@expanded_path).returns(@assets)
+        assert_equal @assets, Hadupils::Extensions::Base.new(@path).assets
+      end
+
+      should "allow manipulation of assets post-expansion" do
+        Hadupils::Extensions::Base.stubs(:gather_assets).returns(@assets)
+        extra = mock()
+        ext = Hadupils::Extensions::Base.new(@path) do
+          assets do |items|
+            # We're just adding the new stuff to the original stuff
+            [items, extra]
+          end
+        end
+        # If the above assets block was applied, we'll see the additional
+        # item there.
+        assert_equal [@assets, extra], ext.assets
+      end
+
+      should 'have an empty hivercs list' do
+        Hadupils::Extensions::Base.stubs(:gather_assets).returns(@assets)
+        assert_equal [], Hadupils::Extensions::Base.new(@path).hivercs
+      end
+    end
+
+    context 'gather_assets' do
+      should 'assemble assets with Hadupils::Assets.assets_in' do
+        path = mock()
+        result = mock()
+        Hadupils::Assets.expects(:assets_in).with(path).returns(result)
+        assert_equal result, Hadupils::Extensions::Base.gather_assets(path)
+      end
+
+      should 'allow manipulation of assets' do
+      end
+
+      should 'produce empty list for a nil path' do
+        Hadupils::Assets.expects(:assets_in).never
+        assert_equal [], Hadupils::Extensions::Base.gather_assets(nil)
+      end
+    end
+  end
+
+  context 'a hiverc' do
+    context 'static wrapper' do
+      setup do
+        @klass = Hadupils::Extensions::HiveRC::Static
+      end
+
+      should 'expand the given path into its path attr' do
+        path = 'foo/bar/blah'
+        assert_equal ::File.expand_path(path), @klass.new(path).path
+      end
+
+      should 'provide a close no-op' do
+        assert_respond_to @klass.new('blah'), :close
+      end
+
+      should 'know how to convert to #hive_opts' do
+        path = 'some/awesome/path'
+        assert_equal ['-i', ::File.expand_path(path)],
+                     @klass.new(path).hive_opts
+      end
+    end
+
+    context 'dynamic wrapper' do
+      setup do
+        @klass = Hadupils::Extensions::HiveRC::Dynamic
+      end
+
+      should 'use Tempfile for its default file_handler' do
+        assert_same ::Tempfile, @klass.file_handler
+      end
+
+      should 'know how to convert to #hive_opts' do
+        obj = @klass.new
+        obj.stubs(:path).returns(path = mock())
+        assert_equal ['-i', obj.path],
+                     obj.hive_opts
+      end
+
+      context 'internal file' do
+        setup do
+          @klass.expects(:file_handler).with().returns(@handler = mock())
+          @handler.expects(:new).with('hadupils-hiverc').returns(@file = mock())
+        end
+
+        should "come from the class' file_handler" do
+          assert_equal @file, @klass.new.file
+        end
+
+        should 'provide the path' do
+          @file.stubs(:path).returns(path = mock())
+          ::File.stubs(:expand_path).with(path).returns(expanded = mock())
+          assert_equal expanded, @klass.new.path
+        end
+
+        should 'close the file on close()' do
+          @file.expects(:close).with()
+          @klass.new.close
+        end
+      end
+
+      context 'write operation' do
+        setup do
+          @hiverc = @klass.new
+          @file = File.open(@hiverc.path, 'r')
+        end
+
+        teardown do
+          @file.close
+          @hiverc.close
+        end
+
+        should 'handle simple text lines' do
+          lines = ['some stuff!', 'but what about...', 'this and this!?!']
+          @hiverc.write(lines)
+          expect = lines.join("\n") + "\n"
+          assert_equal expect, @file.read
+        end
+
+        context 'given assets' do
+          setup do
+            @asset_lines = ['ADD FILE foofoo;',
+                            'ADD ARCHIVE bloobloo.tar.gz;',
+                            'ADD JAR jarjar.jar;']
+            @assets = @asset_lines.collect do |line|
+              m = mock()
+              m.stubs(:hiverc_command).with.returns(line)
+              m
+            end
+          end
+
+          should 'use their hiverc_command for lines' do
+            expected = @asset_lines.join("\n") + "\n"
+            @hiverc.write(@assets)
+            assert_equal expected, @file.read
+          end
+
+          should 'handle intermingled text lines' do
+            text_lines = ['some line one', 'some line two']
+            [@assets, @asset_lines].each do |ary|
+              ary.insert(2, text_lines[1])
+              ary.insert(1, text_lines[0])
+            end
+            expected = @asset_lines.join("\n") + "\n"
+            @hiverc.write(@assets)
+            assert_equal expected, @file.read
+          end
+        end
+      end
+    end
+  end
+
+  context Hadupils::Extensions::Flat do
+    setup do
+      @klass = Hadupils::Extensions::Flat
+    end
+
+    should 'extend Hadupils::Extensions::Base' do
+      # I usually hate testing this sort of thing, but I want to quickly claim
+      # that @klass has the basic behaviors and focus on what's special about it.
+      assert @klass.ancestors.include? Hadupils::Extensions::Base
+    end
+
+    tempdir_context 'for realz' do
+      setup do
+        @tempdir.file(@file = 'a.file')
+        @tempdir.file(@jar = 'a.jar')
+        @tempdir.file(@archive = 'an.archive.tar.gz')
+        @file_line = "ADD FILE #{@tempdir.full_path(@file)};"
+        @jar_line = "ADD JAR #{@tempdir.full_path(@jar)};"
+        @archive_line = "ADD ARCHIVE #{@tempdir.full_path(@archive)};"
+      end
+
+      should 'produce only one hiverc' do
+        hivercs = @klass.new(@tempdir.path).hivercs
+        assert_equal 1, hivercs.size
+      end
+
+      should 'produce a hiverc for the expected assets' do
+        hivercs = @klass.new(@tempdir.path).hivercs
+        expected = "#{@file_line}\n#{@jar_line}\n#{@archive_line}\n"
+        File.open(hivercs[0].path, 'r') do |f|
+          assert_equal expected, f.read
+        end
+      end
+
+      should 'produce a hiverc of a dynamic type' do
+        # This is because I had a bug and was giving the hiverc File
+        # object instead of the dynamic hiverc wrapper object.
+        # Thus it blew up later on.
+        hivercs = @klass.new(@tempdir.path).hivercs
+        assert_kind_of Hadupils::Extensions::HiveRC::Dynamic, hivercs[0]
+      end
+
+      should 'allow manipulation of hiverc items' do
+        extension = @klass.new(@tempdir.path) do
+          hiverc do |assets|
+            assets.insert(1, 'INSERT SOME TEXT HERE')
+            assets << 'FINAL LINE!'
+          end
+        end
+        expected = "#{@file_line}\n" +
+                   "INSERT SOME TEXT HERE\n" +
+                   "#{@jar_line}\n#{@archive_line}\n" +
+                   "FINAL LINE!\n"
+        File.open(extension.hivercs[0].path, 'r') do |f|
+          assert_equal expected, f.read
+        end
+      end
+    end
+  end
+
+  tempdir_context Hadupils::Extensions::Static do
+    setup do
+      @extension = Hadupils::Extensions::Static.new(@tempdir.path)
+    end
+
+    should 'have an empty list of assets from gather_assets' do
+      # These would ordinarily become assets in a dynamic extension.
+      @tempdir.file('some.jar')
+      @tempdir.file('some.tar.gz')
+      @tempdir.file('some.yaml')
+      # but not in this one.
+      assert_equal [], Hadupils::Extensions::Static.new(@tempdir.path).assets
+    end
+
+    should 'have an empty hivercs list when no hiverc file exists' do
+      assert_equal [], @extension.hivercs
+    end
+
+    context 'with a hiverc file' do
+      setup do
+        @hiverc = @tempdir.file('hiverc')
+      end
+
+      should 'have a static HiveRC instance in its hivercs list when a hiverc file exists' do
+        assert_equal [[Hadupils::Extensions::HiveRC::Static, @hiverc.path]],
+                     @extension.hivercs.collect {|h| [h.class, h.path] }
+      end
+    end
+  end
+end

data/test/unit/runners_test.rb

@@ -0,0 +1,80 @@
+class Hadupils::RunnersTest < Test::Unit::TestCase
+  context Hadupils::Runners::Base do
+    setup do
+      @runner = Hadupils::Runners::Base.new(@params = mock())
+    end
+
+    should 'expose initialization params as attr' do
+      assert_equal @params, @runner.params
+    end
+
+    context 'wait!' do
+      setup do
+        @command = [mock(), mock(), mock()]
+        @runner.expects(:command).with.returns(@command)
+        # This will ensure that $? is non-nil
+        system(RbConfig.ruby, '-v')
+      end
+
+      should 'assemble system call via command method' do
+        Kernel.expects(:system).with(*@command).returns(true)
+        $?.stubs(:exitstatus).with.returns(mock())
+        @runner.wait!
+      end
+
+      should 'return 255 when system returns nil' do
+        Kernel.stubs(:system).returns(nil)
+        assert_equal 255, @runner.wait!
+      end
+
+      should 'return Process::Status#exitstatus when non-nil system result' do
+        Kernel.stubs(:system).returns(true)
+        $?.stubs(:exitstatus).with.returns(status = mock())
+        assert_equal status, @runner.wait!
+      end
+    end
+  end
+
+  context Hadupils::Runners::Hive do
+    setup do
+      @klass = Hadupils::Runners::Hive
+    end
+
+    should 'be a runner' do
+      assert_kind_of Hadupils::Runners::Base, @klass.new([])
+    end
+
+    should 'use $HIVE_HOME/bin/hive as the base runner' do
+      ENV.expects(:[]).with('HIVE_HOME').returns(home = mock().to_s)
+      assert_equal ::File.join(home, 'bin', 'hive'),
+                   @klass.base_runner
+    end
+
+    context '#command' do
+      setup do
+        @klass.stubs(:base_runner).returns(@hive_path = mock().to_s + '-hive')
+      end
+
+      should 'provide invocation for bare hive if given empty parameters' do
+        assert_equal [@hive_path], @klass.new([]).command
+      end
+
+      should 'provide invocation for hive with all given parameters' do
+        params = [mock().to_s, mock().to_s, mock().to_s, mock().to_s]
+        assert_equal [@hive_path] + params,
+                     @klass.new(params).command
+      end
+
+      should 'provide args for hive with :hive_opts on supporting params' do
+        p1 = mock()
+        p1.expects(:hive_opts).with.returns(p1_opts = ['-i', mock().to_s])
+        p2 = mock()
+        p2.expects(:hive_opts).with.returns(p2_opts = ['-i', mock().to_s])
+        s1 = mock().to_s
+        s2 = mock().to_s
+        assert_equal [@hive_path, s1] + p1_opts + [s2] + p2_opts,
+                     @klass.new([s1, p1, s2, p2]).command
+      end
+    end
+  end
+end

data/test/unit/search_test.rb

@@ -0,0 +1,83 @@
+class Hadupils::SearchTest < Test::Unit::TestCase
+  tempdir_context 'find_from_dir' do
+    setup do
+      @module = Hadupils::Search
+      @search_name = mock().to_s
+    end
+
+    should 'return nil if requested directory cannot be found' do
+      assert_equal nil, @module.find_from_dir(@search_name, @tempdir.path)
+    end
+
+    should 'should find the directory when it is in the start dir' do
+      p = @tempdir.full_path('blah')
+      Dir.mkdir p
+      assert_equal p, @module.find_from_dir('blah', @tempdir.path)
+    end
+
+    should 'find the directory when it is in a sibling of the start dir' do
+      target = @tempdir.full_path('target-dir')
+      start = @tempdir.full_path('start-dir')
+      [target, start].each {|d| Dir.mkdir(d) }
+      assert_equal target, @module.find_from_dir('target-dir', start)
+    end
+
+    should 'find the directory when it is above the start dir' do
+      d = @tempdir.full_path('flickityflu')
+      Dir.mkdir(d)
+      assert_equal @tempdir.path,
+                   @module.find_from_dir(File.basename(@tempdir.path), d)
+    end
+  end
+
+  context 'find_from_pwd' do
+    setup do
+      @module = Hadupils::Search
+      @pwd = ::Dir.pwd
+      @target = mock()
+    end
+
+    should 'return the path found by find_from_dir for the pwd' do
+      @module.expects(:find_from_dir).with(@target, @pwd).returns(result = mock())
+      assert_equal result, @module.find_from_pwd(@target)
+    end
+
+    should 'return nil when given that by find_from_dir for the pwd' do
+      @module.expects(:find_from_dir).with(@target, @pwd).returns(nil)
+      assert_equal nil, @module.find_from_pwd(@target)
+    end
+  end
+
+  context 'user_config' do
+    setup do
+      @module = Hadupils::Search
+    end
+
+    should 'use ~/conf by default' do
+      assert_equal ::File.expand_path(::File.join('~', 'conf')),
+                   @module.user_config
+    end
+
+    should 'be settable' do
+      assert_equal true, @module.respond_to?(:user_config=)
+    end
+  end
+
+  context 'hadoop_assets' do
+    should 'search for directory specified by #hadoop_assets_name' do
+      Hadupils::Search.expects(:hadoop_assets_name).with.returns(name = mock().to_s)
+      Hadupils::Search.expects(:find_from_pwd).with(name).returns(dir = mock())
+      assert_equal dir, Hadupils::Search.hadoop_assets
+    end
+  end
+
+  context 'hadoop_assets_name' do
+    should 'default to "hadoop-ext"' do
+      assert_equal 'hadoop-ext', Hadupils::Search.hadoop_assets_name
+    end
+
+    should 'be settable' do
+      assert_respond_to Hadupils::Search, :hadoop_assets_name=
+    end
+  end
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: hadupils
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+  prerelease: 
+platform: ruby
+authors:
+- Ethan Rowe
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: should-context
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Provides utilities for dynamic hadoop client environment configuration
+email: ethan@the-rowes.com
+executables:
+- hadupils
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/hadupils/search.rb
+- lib/hadupils/commands.rb
+- lib/hadupils/runners.rb
+- lib/hadupils/extensions.rb
+- lib/hadupils/assets.rb
+- lib/hadupils.rb
+- test/unit/assets_test.rb
+- test/unit/commands_test.rb
+- test/unit/extensions_test.rb
+- test/unit/runners_test.rb
+- test/unit/search_test.rb
+- test/hadupil_test_setup.rb
+- bin/hadupils
+- Rakefile.rb
+- Gemfile.lock
+- README.md
+- Gemfile
+- LICENSE
+- CHANGELOG.md
+homepage: http://github.com/ethanrowe/hadupils
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Provides utilities for dynamic hadoop client environment configuration
+test_files: []