hadupils 0.3.0 → 0.4.0

data/CHANGELOG.md

@@ -38,3 +38,9 @@
   when a streaming query runs.
 * Some misc. utility functions in Hadupils::Util for reading tarballs.
 
+### 0.4.0
+
+* Introduced Hadupils::Extensions::Hive.build_archive
+  Helper method for assembling gzipped archives the contents of which
+  are hive-ext compatible.
+

data/lib/hadupils/extensions/hive.rb

@@ -162,6 +162,71 @@ module Hadupils::Extensions
     def self.assemble_static_extension(path)
       Static.new(path)
     end
+
+    # Writes a gzipped tar archive to +io+, the contents of which
+    # are structured appropriately for use with this class.
+    # 
+    # Provide the static hiverc and any other distributed cache-bound
+    # assets in +dist_assets+, and any auxiliary jars to include in
+    # +aux_jars+.
+    #
+    # This utilizes a system call to +tar+ under the hood, which requires
+    # that it be installed and on your +PATH+.
+    #
+    # You can use any file-like writable thing for +io+, so files,
+    # pipes, etc.
+    #
+    # See this example:
+    #
+    #     File.open('foo.tar.gz', 'w') do |f|
+    #       Hadupils::Extensions::Hive.build_archive f,
+    #                                                ['/tmp/here/blah.jar',
+    #                                                 '/tmp/there/hiverc'],
+    #                                                ['/tmp/elsewhere/foo.jar']
+    #     end
+    #
+    # The following example would produce an archive named "foo.tar.gz",
+    # the contents of which would be:
+    #
+    #     aux-jars/foo.jar
+    #     blah.jar
+    #     hiverc
+    #
+    # Note that it collapses things into two distinct directories, such
+    # that basename collisions are possible.  That's on you to handle
+    # sanely.
+    def self.build_archive(io, dist_assets, aux_jars=nil)
+      dist, aux = [dist_assets, (aux_jars || [])].collect do |files|
+        files.collect do |asset|
+          path = ::File.expand_path(asset)
+          raise "Cannot include directory '#{path}'." if ::File.directory? path
+          path
+        end
+      end
+
+      require 'tempfile'
+      require 'fileutils'
+      ::Dir.mktmpdir do |workdir|
+        basenames = dist.collect do |src|
+          FileUtils.cp src, File.join(workdir, File.basename(src))
+          File.basename src
+        end
+
+        if aux.length > 0
+          basenames << AUX_PATH
+          aux_dir = File.join(workdir, AUX_PATH)
+          Dir.mkdir aux_dir
+          aux.each do |src|
+            FileUtils.cp src, File.join(aux_dir, File.basename(src))
+          end
+        end
+
+        ::Dir.chdir(workdir) do |p|
+          Kernel.system 'tar', 'cz', *basenames, :out => io
+        end
+      end
+      true
+    end
   end
 
   # Collection class for filesystem-based Hive extensions

data/test/unit/extensions/hive_test.rb

@@ -106,6 +106,51 @@ class Hadupils::Extensions::HiveTest < Test::Unit::TestCase
     end
   end
 
+  shared_context :assembles_compressed_archive do
+    setup do
+      @unpack_dir = @tempdir.full_path('unpack')
+      Dir.mkdir @unpack_dir
+
+      @pipe_read, @pipe_write = IO.pipe
+      Dir.chdir @unpack_dir do |p|
+        @tar_pid = Kernel.spawn 'tar', 'xz', :in => @pipe_read
+      end
+    end
+
+    should 'assemble a hive-ext-compatible compressed archive' do
+      @pipe_write.close
+      Process.wait @tar_pid
+      dist, aux = [[@unpack_dir, ['aux-jars']],
+                   [File.join(@unpack_dir, 'aux-jars'), []]].collect do |dir, excl|
+        excl += ['.', '..']
+        begin
+          Dir.entries(dir).inject({}) do |a, entry|
+            if not excl.include?(entry)
+              a[entry] = File.open(File.join(dir, entry), 'r') {|f| f.read}
+            end
+            a
+          end
+        rescue Exception => e
+          {}
+        end
+      end
+
+      dist_exp = {
+        File.basename(@dist_jar_a) => @dist_jar_a_content,
+        File.basename(@dist_jar_b) => @dist_jar_b_content,
+        File.basename(@hiverc) => @hiverc_content,
+      }
+
+      aux_exp = {
+        File.basename(@aux_jar_a) => @aux_jar_a_content,
+        File.basename(@aux_jar_b) => @aux_jar_b_content,
+      }
+
+      assert_equal dist_exp, dist
+      assert_equal aux_exp, aux
+    end
+  end
+
   tempdir_context Hadupils::Extensions::Hive do
     context 'with auxiliary jars' do
       provide_hive_ext
@@ -188,6 +233,92 @@ class Hadupils::Extensions::HiveTest < Test::Unit::TestCase
         end
       end
     end
+
+    context 'build_archive singleton method' do
+      require 'stringio'
+
+      setup do
+        ::Dir.mkdir(@some_dir = @tempdir.full_path('some_stuff'))
+        ::Dir.mkdir(@other_dir = @tempdir.full_path('other_stuff'))
+        @dist_jar_a = ::File.open(::File.join(@some_dir, 'a.jar'), 'w') do |f|
+          f.write(@dist_jar_a_content = 'dist jar a')
+          f.path
+        end
+
+        @aux_jar_a = ::File.open(::File.join(@some_dir, 'auxa.jar'), 'w') do |f|
+          f.write(@aux_jar_a_content = 'aux jar a')
+          f.path
+        end
+
+        @hiverc = @tempdir.file('hiverc') {|f| f.write(@hiverc_content = 'hiverc stuff'); f.path}
+
+        @dist_jar_b = ::File.open(::File.join(@other_dir, 'b.jar'), 'w') do |f|
+          f.write(@dist_jar_b_content = 'dist jar b')
+          f.path
+        end
+
+        @aux_jar_b = ::File.open(::File.join(@other_dir, 'auxb.jar'), 'w') do |f|
+          f.write(@aux_jar_b_content = 'aux jar b')
+          f.path
+        end
+
+        @mod = Hadupils::Extensions::Hive
+      end
+
+      should 'barf on a dist directory' do
+        # assert_raise ain't doing this for me...
+        e = nil
+        begin
+          @mod.build_archive StringIO.new(''), [@tempdir.path]
+        rescue Exception => e
+        end
+
+        assert_instance_of RuntimeError, e
+        assert_equal "Cannot include directory '#{@tempdir.path}'.", e.message
+      end
+
+      should 'barf on an aux directory' do
+        # assert_raise ain't doing this for me...
+        e = nil
+        begin
+          @mod.build_archive StringIO.new(''), [], [@tempdir.path]
+        rescue Exception => e
+        end
+
+        assert_instance_of RuntimeError, e
+        assert_equal "Cannot include directory '#{@tempdir.path}'.", e.message
+      end
+
+      context 'given absolute paths' do
+        assembles_compressed_archive
+
+        setup do
+          @mod.build_archive @pipe_write,
+                             [@dist_jar_a, @dist_jar_b, @hiverc],
+                             [@aux_jar_a, @aux_jar_b]
+        end
+      end
+
+      context 'given relative paths' do
+        assembles_compressed_archive
+
+        setup do
+          chars = @tempdir.path.length + 1 # +1 for trailing slash
+
+          dist = [@dist_jar_a, @dist_jar_b, @hiverc].collect do |path|
+            path[chars..-1]
+          end
+
+          aux = [@aux_jar_a, @aux_jar_b].collect do |path|
+            path[chars..-1]
+          end
+
+          Dir.chdir(@tempdir.path) do |p|
+            @mod.build_archive @pipe_write, dist, aux
+          end
+        end
+      end
+    end
   end
 
   tempdir_context Hadupils::Extensions::HiveSet do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hadupils
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors: