hadupils 0.2.0 → 0.3.0

data/CHANGELOG.md

@@ -30,3 +30,11 @@
   parameter (custom support for pre-1.9 ruby, the Kernel.system call
   of which does not handle such things)
 
+### 0.3.0
+
+* Introduced Hadupils::Extensions::FlatArchivePath
+* The "hive" command uses a FlatArchivePath for hadoop-ext instead
+  of Flat, so PATH will include bin dirs of any hadoop-ext archives
+  when a streaming query runs.
+* Some misc. utility functions in Hadupils::Util for reading tarballs.
+

data/lib/hadupils/commands.rb

@@ -25,7 +25,7 @@ module Hadupils::Commands
 
   module HadoopExt
     def hadoop_ext
-      @hadoop_ext ||= Hadupils::Extensions::Flat.new(Hadupils::Search.hadoop_assets)
+      @hadoop_ext ||= Hadupils::Extensions::FlatArchivePath.new(Hadupils::Search.hadoop_assets)
     end
   end
 

data/lib/hadupils/extensions.rb

@@ -129,8 +129,12 @@ module Hadupils::Extensions
       [@hiverc]
     end
 
+    def default_hiverc_items
+      @assets.dup
+    end
+
     def assemble_hiverc
-      assets = @assets
+      assets = default_hiverc_items
       if @hiverc_block
         assets = @hiverc_block.call(assets.dup)
       end
@@ -140,6 +144,36 @@ module Hadupils::Extensions
     end
   end
 
+  class FlatArchivePath < Flat
+    def archives_for_path_env
+      @assets.find_all do |a|
+        if a.kind_of? Hadupils::Assets::Archive
+          begin
+            Hadupils::Util.archive_has_directory?(a.path, 'bin')
+          rescue
+            false
+          end
+        else
+          false
+        end
+      end
+    end
+
+    def default_hiverc_items
+      items = super
+      archs = archives_for_path_env
+      if archs.length > 0
+        items << self.class.assemble_path_env(archs)
+      end
+      items
+    end
+
+    def self.assemble_path_env(archives)
+      paths = archives.collect {|a| "$(pwd)/#{ a.name }/bin" }
+      "SET mapred.child.env = PATH=#{ paths.join(':') }:$PATH;"
+    end
+  end
+
   class Static < Base
     def self.gather_assets(path)
       []

data/lib/hadupils/util.rb

@@ -0,0 +1,25 @@
+module Hadupils::Util
+  def self.read_archive(archive_path)
+    require 'rubygems/package'
+    require 'zlib'
+    Zlib::GzipReader.open(archive_path) do |zlib|
+      Gem::Package::TarReader.new(zlib) do |tar|
+        tar.rewind
+        yield tar
+      end
+    end
+  end
+
+  def self.archive_has_directory?(archive_path, directory)
+    directory = directory + '/' unless directory.end_with?('/')
+    targets = [directory[0..-2], directory]
+    found = false
+    read_archive(archive_path) do |arch|
+      arch.each do |entry|
+        found = (entry.directory? and targets.include?(entry.full_name))
+        break if found
+      end
+    end
+    found
+  end
+end

data/lib/hadupils.rb

@@ -7,3 +7,4 @@ require 'hadupils/commands'
 require 'hadupils/extensions'
 require 'hadupils/runners'
 require 'hadupils/search'
+require 'hadupils/util'

data/test/unit/commands_test.rb

@@ -44,9 +44,9 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
         assert_equal result, @klass.run(params)
       end
 
-      should 'have a Flat extension based on a search for hadoop-ext' do
+      should 'have a FlatArchivePath extension based on a search for hadoop-ext' do
         Hadupils::Search.expects(:hadoop_assets).with.returns(assets = mock())
-        Hadupils::Extensions::Flat.expects(:new).with(assets).returns(extension = mock())
+        Hadupils::Extensions::FlatArchivePath.expects(:new).with(assets).returns(extension = mock())
         cmd = @klass.new
         assert_equal extension, cmd.hadoop_ext
         # This should cause failure if the previous result wasn't

data/test/unit/extensions/base_test.rb

@@ -231,6 +231,48 @@ class Hadupils::ExtensionsTest < Test::Unit::TestCase
           assert_equal expected, f.read
         end
       end
+
+      context 'ArchivePath extension' do
+        setup do
+          require 'rubygems/package'
+          require 'zlib'
+          @klass = Hadupils::Extensions::FlatArchivePath
+          @bin_archives = %w(one two).collect do |name|
+            @tempdir.file("zarchive-with-bin-#{name}.tar.gz") do |f|
+              Zlib::GzipWriter.open(f) do |zlib|
+                buffer = StringIO.new('')
+                Gem::Package::TarWriter.new(buffer) do |writer|
+                  writer.add_file("#{name}.txt", '644'.oct) {|x| x.write('blah')}
+                  writer.mkdir('bin', '755'.oct)
+                end
+                zlib.write buffer.string
+              end
+              f.path
+            end
+          end
+
+          @bin_archive_lines = @bin_archives.collect {|path| "ADD ARCHIVE #{path};"}.join("\n")
+        end
+
+        should 'produce a hiverc for the expected assets and archives with bins added to PATH' do
+          path_prefix = @bin_archives.collect {|p| "$(pwd)/#{::File.basename(p)}/bin" }.join(':')
+          expected = "#{@file_line}\n#{@jar_line}\n#{@archive_line}\n" +
+                     "#{@bin_archive_lines}\n" +
+                     "SET mapred.child.env = PATH=#{path_prefix}:$PATH;\n" +
+                     (last_line = 'SET last.line = thisone;') + "\n"
+          ext = @klass.new(@tempdir.path) do
+            hiverc do |items|
+              items << last_line
+            end
+          end
+          hivercs = ext.hivercs
+          assert_kind_of Hadupils::Extensions::HiveRC::Dynamic, hivercs[0]
+          assert_equal 1, hivercs.length
+          ::File.open(hivercs[0].path, 'r') do |f|
+            assert_equal expected, f.read
+          end
+        end
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hadupils
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -88,6 +88,7 @@ files:
 - lib/hadupils/extensions/hive.rb
 - lib/hadupils/extensions.rb
 - lib/hadupils/assets.rb
+- lib/hadupils/util.rb
 - lib/hadupils.rb
 - test/unit/assets_test.rb
 - test/unit/commands_test.rb