hadupils 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.md +13 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +16 -0
- data/LICENSE +20 -0
- data/README.md +4 -0
- data/Rakefile.rb +13 -0
- data/bin/hadupils +8 -0
- data/lib/hadupils/assets.rb +57 -0
- data/lib/hadupils/commands.rb +48 -0
- data/lib/hadupils/extensions.rb +162 -0
- data/lib/hadupils/runners.rb +45 -0
- data/lib/hadupils/search.rb +49 -0
- data/lib/hadupils.rb +9 -0
- data/test/hadupil_test_setup.rb +50 -0
- data/test/unit/assets_test.rb +166 -0
- data/test/unit/commands_test.rb +135 -0
- data/test/unit/extensions_test.rb +266 -0
- data/test/unit/runners_test.rb +80 -0
- data/test/unit/search_test.rb +83 -0
- metadata +129 -0
data/CHANGELOG.md
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
|
2
|
+
### 0.1.0
|
3
|
+
|
4
|
+
* Basic functionality for representing a command, extensions,
|
5
|
+
assets, runners.
|
6
|
+
* A hive runner enforcing user config and hadoop-ext extension
|
7
|
+
* A hadupils executable for entering the command model
|
8
|
+
|
9
|
+
### 0.1.1
|
10
|
+
|
11
|
+
* Removed evil rubygems requirement from executable
|
12
|
+
* Added this glorious changelog
|
13
|
+
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2013 Ethan Rowe
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
data/Rakefile.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'rake'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'rake/clean'
|
4
|
+
|
5
|
+
CLOBBER.include('hadupils-*.gem')
|
6
|
+
|
7
|
+
Rake::TestTask.new do |t|
|
8
|
+
t.pattern = 'test/**/*_test.rb'
|
9
|
+
t.libs = ['test', 'lib']
|
10
|
+
# This should initialize the environment properly.
|
11
|
+
t.ruby_opts << '-rhadupil_test_setup'
|
12
|
+
end
|
13
|
+
|
data/bin/hadupils
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
module Hadupils::Assets
|
2
|
+
class File
|
3
|
+
attr_reader :name, :path
|
4
|
+
|
5
|
+
def self.determine_name(path)
|
6
|
+
::File.basename(path)
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.hiverc_type
|
10
|
+
:FILE
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(path)
|
14
|
+
@path = path
|
15
|
+
@name = self.class.determine_name(path)
|
16
|
+
end
|
17
|
+
|
18
|
+
def hidden?
|
19
|
+
name[0] == '.'
|
20
|
+
end
|
21
|
+
|
22
|
+
def hiverc_command
|
23
|
+
"ADD #{self.class.hiverc_type} #{path};"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
class Jar < File
|
28
|
+
def self.hiverc_type
|
29
|
+
:JAR
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class Archive < File
|
34
|
+
def self.hiverc_type
|
35
|
+
:ARCHIVE
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.asset_for(path)
|
40
|
+
return Archive.new(path) if path[-7..-1] == '.tar.gz'
|
41
|
+
return Jar.new(path) if path[-4..-1] == '.jar'
|
42
|
+
return File.new(path)
|
43
|
+
end
|
44
|
+
|
45
|
+
SKIP_NAMES = ['.', '..']
|
46
|
+
|
47
|
+
# Walks the top-level members of the stated directory and
|
48
|
+
# returns an array containing appropriate an HadoopAsset::*
|
49
|
+
# instance for each.
|
50
|
+
def self.assets_in(directory)
|
51
|
+
path = ::File.expand_path(directory)
|
52
|
+
::Dir.entries(path).sort.inject([]) do |accum, entry|
|
53
|
+
accum << asset_for(::File.join(path, entry)) if not SKIP_NAMES.include? entry
|
54
|
+
accum
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module Hadupils::Commands
|
2
|
+
def self.run(command, params)
|
3
|
+
handler = handler_for command
|
4
|
+
handler.run params
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.handler_for(command)
|
8
|
+
@handlers and @handlers[command.downcase.to_sym]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.register_handler(command, handler)
|
12
|
+
@handlers ||= {}
|
13
|
+
@handlers[command.downcase.to_sym] = handler
|
14
|
+
end
|
15
|
+
|
16
|
+
class SimpleCommand
|
17
|
+
def self.run(params)
|
18
|
+
self.new.run params
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module HadoopExt
|
23
|
+
def hadoop_ext
|
24
|
+
@hadoop_ext ||= Hadupils::Extensions::Flat.new(Hadupils::Search.hadoop_assets)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
module UserConf
|
29
|
+
def user_config
|
30
|
+
@user_config ||= Hadupils::Extensions::Static.new(Hadupils::Search.user_config)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class Hive < SimpleCommand
|
35
|
+
include HadoopExt
|
36
|
+
include UserConf
|
37
|
+
|
38
|
+
def assemble_parameters(parameters)
|
39
|
+
user_config.hivercs + hadoop_ext.hivercs + parameters
|
40
|
+
end
|
41
|
+
|
42
|
+
def run(parameters)
|
43
|
+
Hadupils::Runners::Hive.run assemble_parameters(parameters)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
register_handler :hive, Hive
|
48
|
+
end
|
@@ -0,0 +1,162 @@
|
|
1
|
+
module Hadupils::Extensions
|
2
|
+
# Tools for managing hive initialization files ("hiverc").
|
3
|
+
module HiveRC
|
4
|
+
module HiveOpt
|
5
|
+
def hive_opts
|
6
|
+
['-i', path]
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
# Wraps an extant hive initialization file, and providing
|
11
|
+
# an interface compatible with the critical parts of the
|
12
|
+
# Dynamic sibling class so they may be used interchangeably
|
13
|
+
# by runners when determining hive options.
|
14
|
+
class Static
|
15
|
+
attr_reader :path
|
16
|
+
|
17
|
+
include HiveOpt
|
18
|
+
|
19
|
+
# Given a path, expands it ti
|
20
|
+
def initialize(path)
|
21
|
+
@path = ::File.expand_path(path)
|
22
|
+
end
|
23
|
+
|
24
|
+
def close
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# Manages dynamic hive initialization files, assembling a temporary file
|
29
|
+
# and understanding how to write assets/lines into the initialization file for
|
30
|
+
# use with hive's -i option.
|
31
|
+
class Dynamic
|
32
|
+
attr_reader :file
|
33
|
+
|
34
|
+
include HiveOpt
|
35
|
+
require 'tempfile'
|
36
|
+
|
37
|
+
# This will allow us to change what handles the dynamic files.
|
38
|
+
def self.file_handler=(handler)
|
39
|
+
@file_handler = handler
|
40
|
+
end
|
41
|
+
|
42
|
+
# The class to use for creating the files; defaults to ::Tempfile
|
43
|
+
def self.file_handler
|
44
|
+
@file_handler || ::Tempfile
|
45
|
+
end
|
46
|
+
|
47
|
+
# Sets up a wrapped file, using the class' file_handler,
|
48
|
+
def initialize
|
49
|
+
@file = self.class.file_handler.new('hadupils-hiverc')
|
50
|
+
end
|
51
|
+
|
52
|
+
def path
|
53
|
+
::File.expand_path @file.path
|
54
|
+
end
|
55
|
+
|
56
|
+
def close
|
57
|
+
@file.close
|
58
|
+
end
|
59
|
+
|
60
|
+
# Writes the items to the file, using #hiverc_command on each item that
|
61
|
+
# responds to it (Hadupils::Assets::* instances) and #to_s on the rest.
|
62
|
+
# Separates lines by newline, and provides a trailing newline. However,
|
63
|
+
# the items are responsible for ensuring the proper terminating semicolon.
|
64
|
+
# The writes are flushed to the underlying file immediately afterward.
|
65
|
+
def write(items)
|
66
|
+
lines = items.collect do |item|
|
67
|
+
if item.respond_to? :hiverc_command
|
68
|
+
item.hiverc_command
|
69
|
+
else
|
70
|
+
item.to_s
|
71
|
+
end
|
72
|
+
end
|
73
|
+
@file.write(lines.join("\n") + "\n")
|
74
|
+
@file.flush
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
class EvalProxy
|
80
|
+
def initialize(scope)
|
81
|
+
@scope = scope
|
82
|
+
end
|
83
|
+
|
84
|
+
def assets(&block)
|
85
|
+
@scope.instance_eval do
|
86
|
+
@assets_block = block
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def hiverc(&block)
|
91
|
+
@scope.instance_eval do
|
92
|
+
@hiverc_block = block
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
class Base
|
98
|
+
attr_reader :assets, :path
|
99
|
+
|
100
|
+
def initialize(directory, &block)
|
101
|
+
if block_given?
|
102
|
+
EvalProxy.new(self).instance_eval &block
|
103
|
+
end
|
104
|
+
@path = ::File.expand_path(directory) unless directory.nil?
|
105
|
+
@assets = merge_assets(self.class.gather_assets(@path))
|
106
|
+
end
|
107
|
+
|
108
|
+
def merge_assets(assets)
|
109
|
+
return @assets_block.call(assets) if @assets_block
|
110
|
+
assets
|
111
|
+
end
|
112
|
+
|
113
|
+
def hivercs
|
114
|
+
[]
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.gather_assets(directory)
|
118
|
+
if not directory.nil?
|
119
|
+
Hadupils::Assets.assets_in(directory)
|
120
|
+
else
|
121
|
+
[]
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
class Flat < Base
|
127
|
+
def hivercs
|
128
|
+
@hiverc ||= assemble_hiverc
|
129
|
+
[@hiverc]
|
130
|
+
end
|
131
|
+
|
132
|
+
def assemble_hiverc
|
133
|
+
assets = @assets
|
134
|
+
if @hiverc_block
|
135
|
+
assets = @hiverc_block.call(assets.dup)
|
136
|
+
end
|
137
|
+
hiverc = Hadupils::Extensions::HiveRC::Dynamic.new
|
138
|
+
hiverc.write(assets)
|
139
|
+
hiverc
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
class Static < Base
|
144
|
+
def self.gather_assets(path)
|
145
|
+
[]
|
146
|
+
end
|
147
|
+
|
148
|
+
def hiverc_path
|
149
|
+
::File.join(path, 'hiverc')
|
150
|
+
end
|
151
|
+
|
152
|
+
def hiverc?
|
153
|
+
::File.file? hiverc_path
|
154
|
+
end
|
155
|
+
|
156
|
+
def hivercs
|
157
|
+
r = []
|
158
|
+
r << Hadupils::Extensions::HiveRC::Static.new(hiverc_path) if hiverc?
|
159
|
+
r
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Hadupils::Runners
|
2
|
+
class Base
|
3
|
+
attr_reader :params, :last_result, :last_status
|
4
|
+
|
5
|
+
def initialize(params)
|
6
|
+
@params = params
|
7
|
+
end
|
8
|
+
|
9
|
+
def command; end
|
10
|
+
|
11
|
+
def wait!
|
12
|
+
@last_result = Kernel.system(*command)
|
13
|
+
@last_status = $?
|
14
|
+
if @last_result.nil?
|
15
|
+
255
|
16
|
+
else
|
17
|
+
@last_status.exitstatus
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.run(params)
|
22
|
+
self.new(params).wait!
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
class Hive < Base
|
27
|
+
def self.base_runner
|
28
|
+
@base_runner || ::File.join(ENV['HIVE_HOME'], 'bin', 'hive')
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.base_runner=(runner_path)
|
32
|
+
@base_runner = runner_path
|
33
|
+
end
|
34
|
+
|
35
|
+
def command
|
36
|
+
items = params.inject([self.class.base_runner]) do |result, param|
|
37
|
+
if param.respond_to? :hive_opts
|
38
|
+
result + param.hive_opts
|
39
|
+
else
|
40
|
+
result << param
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Hadupils::Search
|
2
|
+
# Searches for directory containing a subdirectory `name`, starting at
|
3
|
+
# the specified `start` directory and walking upwards until it can go
|
4
|
+
# no farther. On first match, the absolute path to that subdirectory
|
5
|
+
# is returned. If not found, returns nil.
|
6
|
+
def self.find_from_dir(name, start)
|
7
|
+
curr = ::File.expand_path(start)
|
8
|
+
last = nil
|
9
|
+
while curr != last
|
10
|
+
p = ::File.join(curr, name)
|
11
|
+
return p if ::File.directory? p
|
12
|
+
last = curr
|
13
|
+
curr = ::File.dirname(curr)
|
14
|
+
end
|
15
|
+
nil
|
16
|
+
end
|
17
|
+
|
18
|
+
# Performs a `find_from_dir` starting at the present working directory.
|
19
|
+
def self.find_from_pwd(name)
|
20
|
+
find_from_dir(name, ::Dir.pwd)
|
21
|
+
end
|
22
|
+
|
23
|
+
# The directory for user-specific configuration files.
|
24
|
+
def self.user_config
|
25
|
+
@user_config || ::File.expand_path(::File.join('~', 'conf'))
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.user_config=(path)
|
29
|
+
@user_config = path
|
30
|
+
end
|
31
|
+
|
32
|
+
# The basename to use when looking for hadoop assets from pwd.
|
33
|
+
def self.hadoop_assets_name
|
34
|
+
@hadoop_assets_name || 'hadoop-ext'
|
35
|
+
end
|
36
|
+
|
37
|
+
# Set the basename to use when looking for hadoop assets from pwd.
|
38
|
+
def self.hadoop_assets_name=(basename)
|
39
|
+
@hadoop_assets_name = basename
|
40
|
+
end
|
41
|
+
|
42
|
+
# A search for #hadoop_assets_name from the pwd.
|
43
|
+
# The default behavior is to look for a subdir named "hadoop-ext",
|
44
|
+
# starting from the current working directory and walking upwards until
|
45
|
+
# a match is found or the file system root is encountered.
|
46
|
+
def self.hadoop_assets
|
47
|
+
find_from_pwd(hadoop_assets_name)
|
48
|
+
end
|
49
|
+
end
|
data/lib/hadupils.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'bundler'
|
2
|
+
Bundler.setup
|
3
|
+
require 'test/unit'
|
4
|
+
require 'shoulda-context'
|
5
|
+
require 'mocha/setup'
|
6
|
+
require 'tempfile'
|
7
|
+
require 'hadupils'
|
8
|
+
|
9
|
+
# Add tempdir niceties to Test::Unit::TestCase
|
10
|
+
# on top of the shoulda-context stuff.
|
11
|
+
class Test::Unit::TestCase
|
12
|
+
class DirWrapper
|
13
|
+
attr_reader :path
|
14
|
+
|
15
|
+
def initialize(path)
|
16
|
+
@path = path
|
17
|
+
end
|
18
|
+
|
19
|
+
def full_path(path)
|
20
|
+
::File.expand_path(::File.join(@path, path))
|
21
|
+
end
|
22
|
+
|
23
|
+
def file(path)
|
24
|
+
path = full_path(path)
|
25
|
+
if block_given?
|
26
|
+
::File.open(path, 'w') do |f|
|
27
|
+
yield f
|
28
|
+
end
|
29
|
+
else
|
30
|
+
::File.new(path, 'w')
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.tempdir_context(name, &block)
|
36
|
+
context name do
|
37
|
+
setup do
|
38
|
+
@tempdir = Test::Unit::TestCase::DirWrapper.new(::File.expand_path(::Dir.mktmpdir))
|
39
|
+
end
|
40
|
+
|
41
|
+
teardown do
|
42
|
+
FileUtils.remove_entry @tempdir.path
|
43
|
+
end
|
44
|
+
|
45
|
+
# Instance_eval instead of simple yield to ensure it happens in the Context object
|
46
|
+
# and not in the test case subclass.
|
47
|
+
instance_eval &block
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,166 @@
|
|
1
|
+
class AssetsTest < Test::Unit::TestCase
|
2
|
+
context 'a file' do
|
3
|
+
setup do
|
4
|
+
@path = '/foo/bar/some_file.blah'
|
5
|
+
@asset = Hadupils::Assets::File.new(@path)
|
6
|
+
end
|
7
|
+
|
8
|
+
should 'have a path' do
|
9
|
+
assert_equal @path, @asset.path
|
10
|
+
end
|
11
|
+
|
12
|
+
should 'have a name' do
|
13
|
+
assert_equal ::File.basename(@path), @asset.name
|
14
|
+
end
|
15
|
+
|
16
|
+
should 'have an ADD FILE hiverc command' do
|
17
|
+
assert_equal "ADD FILE #{@path};", @asset.hiverc_command
|
18
|
+
end
|
19
|
+
|
20
|
+
should 'not be hidden' do
|
21
|
+
assert_equal false, @asset.hidden?
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
context 'a jar' do
|
26
|
+
setup do
|
27
|
+
@path = '/blah/blargh/../foo/blee/something.jar'
|
28
|
+
@asset = Hadupils::Assets::Jar.new(@path)
|
29
|
+
end
|
30
|
+
|
31
|
+
should 'have a path' do
|
32
|
+
assert_equal @path, @asset.path
|
33
|
+
end
|
34
|
+
|
35
|
+
should 'have a name' do
|
36
|
+
assert_equal ::File.basename(@path), @asset.name
|
37
|
+
end
|
38
|
+
|
39
|
+
should 'have an ADD JAR hiverc command' do
|
40
|
+
assert_equal "ADD JAR #{@path};", @asset.hiverc_command
|
41
|
+
end
|
42
|
+
|
43
|
+
should 'not be hidden' do
|
44
|
+
assert_equal false, @asset.hidden?
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
context 'a tarball' do
|
49
|
+
setup do
|
50
|
+
@path = '/blah/blargh/../foo/blee/something.tar.gz'
|
51
|
+
@asset = Hadupils::Assets::Archive.new(@path)
|
52
|
+
end
|
53
|
+
|
54
|
+
should 'have a path' do
|
55
|
+
assert_equal @path, @asset.path
|
56
|
+
end
|
57
|
+
|
58
|
+
should 'have a name' do
|
59
|
+
assert_equal ::File.basename(@path), @asset.name
|
60
|
+
end
|
61
|
+
|
62
|
+
should 'have an ADD ARCHIVE hiverc command' do
|
63
|
+
assert_equal "ADD ARCHIVE #{@path};", @asset.hiverc_command
|
64
|
+
end
|
65
|
+
|
66
|
+
should 'not be hidden' do
|
67
|
+
assert_equal false, @asset.hidden?
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'a hidden file' do
|
72
|
+
should 'have a hidden File asset' do
|
73
|
+
assert_equal true, Hadupils::Assets::File.new('/blah/blee/.foo.bar.txt').hidden?
|
74
|
+
end
|
75
|
+
|
76
|
+
should 'have a hidden Archive asset' do
|
77
|
+
assert_equal true, Hadupils::Assets::Archive.new('/floo/flam/.foo.tar.gz').hidden?
|
78
|
+
end
|
79
|
+
|
80
|
+
should 'have a hidden Jar asset' do
|
81
|
+
assert_equal true, Hadupils::Assets::Jar.new('/flibbidy/blop/.foo.bar.jar').hidden?
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
context 'asset_for' do
|
86
|
+
context 'given a file of no particular extension' do
|
87
|
+
setup do
|
88
|
+
@path = '/some/special/file.path'
|
89
|
+
@asset = Hadupils::Assets.asset_for(@path)
|
90
|
+
end
|
91
|
+
|
92
|
+
should 'produce a Hadupils::Assets::File' do
|
93
|
+
assert_same Hadupils::Assets::File, @asset.class
|
94
|
+
end
|
95
|
+
|
96
|
+
should 'pass the path through' do
|
97
|
+
assert_equal @path, @asset.path
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
context 'given a file with a .jar extension' do
|
102
|
+
setup do
|
103
|
+
@path = '/some/great/magical-1.7.9.jar'
|
104
|
+
@asset = Hadupils::Assets.asset_for(@path)
|
105
|
+
end
|
106
|
+
|
107
|
+
should 'product a Hadupils::Assets::Jar' do
|
108
|
+
assert_same Hadupils::Assets::Jar, @asset.class
|
109
|
+
end
|
110
|
+
|
111
|
+
should 'pass the path through' do
|
112
|
+
assert_equal @path, @asset.path
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
context 'given a file with a .tar.gz extension' do
|
117
|
+
setup do
|
118
|
+
@path = '/some/freaking/awesome.tar.gz'
|
119
|
+
@asset = Hadupils::Assets.asset_for(@path)
|
120
|
+
end
|
121
|
+
|
122
|
+
should 'produce a Hadupils::Assets::Archive' do
|
123
|
+
assert_same Hadupils::Assets::Archive, @asset.class
|
124
|
+
end
|
125
|
+
|
126
|
+
should 'pass the path through' do
|
127
|
+
assert_equal @path, @asset.path
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
tempdir_context 'a directory with files' do
|
133
|
+
setup do
|
134
|
+
@tempdir.file(@archive = 'Awesome-archive.tar.gz')
|
135
|
+
@tempdir.file(@jar = 'jarrIE.jar')
|
136
|
+
@tempdir.file(@file = 'sumyummy.yaml')
|
137
|
+
@matches = {@archive => Hadupils::Assets::Archive,
|
138
|
+
@jar => Hadupils::Assets::Jar,
|
139
|
+
@file => Hadupils::Assets::File}
|
140
|
+
end
|
141
|
+
|
142
|
+
context 'given to Hadupils::Assets.assets_in' do
|
143
|
+
setup do
|
144
|
+
@received = Hadupils::Assets.assets_in(@tempdir.path)
|
145
|
+
end
|
146
|
+
|
147
|
+
should 'get assets in lexicographic order' do
|
148
|
+
assert_equal @matches.keys.sort, (@received.collect {|a| a.name})
|
149
|
+
end
|
150
|
+
|
151
|
+
should 'get assets of appropriate type' do
|
152
|
+
type_map = @received.inject({}) {|hash, asset| hash[asset.name] = asset.class; hash}
|
153
|
+
assert_equal @matches, type_map
|
154
|
+
end
|
155
|
+
|
156
|
+
should 'get assets with expanded paths' do
|
157
|
+
path_map = @received.inject({}) {|hash, asset| hash[asset.name] = asset.path; hash}
|
158
|
+
expected = @matches.keys.inject({}) do |hash, name|
|
159
|
+
hash[name] = @tempdir.full_path(name)
|
160
|
+
hash
|
161
|
+
end
|
162
|
+
assert_equal expected, path_map
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
class Hadupils::CommandsTest < Test::Unit::TestCase
|
2
|
+
context Hadupils::Commands do
|
3
|
+
context 'run singleton method' do
|
4
|
+
should 'pass trailing params to #run method of handler identified by first param' do
|
5
|
+
Hadupils::Commands.expects(:handler_for).with(cmd = mock()).returns(handler = mock())
|
6
|
+
handler.expects(:run).with(params = [mock(), mock(), mock()]).returns(result = mock())
|
7
|
+
assert_equal result, Hadupils::Commands.run(cmd, params)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
context 'Hive' do
|
12
|
+
setup do
|
13
|
+
@klass = Hadupils::Commands::Hive
|
14
|
+
end
|
15
|
+
|
16
|
+
should 'register with :hive name' do
|
17
|
+
assert_same @klass, Hadupils::Commands.handler_for(:hive)
|
18
|
+
assert_same @klass, Hadupils::Commands.handler_for(:HivE)
|
19
|
+
assert_same @klass, Hadupils::Commands.handler_for('hive')
|
20
|
+
assert_same @klass, Hadupils::Commands.handler_for('hIVe')
|
21
|
+
end
|
22
|
+
|
23
|
+
should 'have a #run singleton method that dispatches to an instance #run' do
|
24
|
+
@klass.expects(:new).with.returns(instance = mock())
|
25
|
+
instance.expects(:run).with(params = mock()).returns(result = mock())
|
26
|
+
assert_equal result, @klass.run(params)
|
27
|
+
end
|
28
|
+
|
29
|
+
should 'have a Flat extension based on a search for hadoop-ext' do
|
30
|
+
Hadupils::Search.expects(:hadoop_assets).with.returns(assets = mock())
|
31
|
+
Hadupils::Extensions::Flat.expects(:new).with(assets).returns(extension = mock())
|
32
|
+
cmd = @klass.new
|
33
|
+
assert_equal extension, cmd.hadoop_ext
|
34
|
+
# This should cause failure if the previous result wasn't
|
35
|
+
# cached internally (by breaking expectations).
|
36
|
+
cmd.hadoop_ext
|
37
|
+
end
|
38
|
+
|
39
|
+
should 'have a Static extensions based on user config' do
|
40
|
+
Hadupils::Search.expects(:user_config).with.returns(conf = mock())
|
41
|
+
Hadupils::Extensions::Static.expects(:new).with(conf).returns(extension = mock())
|
42
|
+
cmd = @klass.new
|
43
|
+
assert_equal extension, cmd.user_config
|
44
|
+
# Fails on expectations if previous result wasn't cached.
|
45
|
+
cmd.user_config
|
46
|
+
end
|
47
|
+
|
48
|
+
context '#run' do
|
49
|
+
setup do
|
50
|
+
@command = @klass.new
|
51
|
+
@command.stubs(:user_config).with.returns(@user_config = mock())
|
52
|
+
@command.stubs(:hadoop_ext).with.returns(@hadoop_ext = mock())
|
53
|
+
@runner_class = Hadupils::Runners::Hive
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'with user config and hadoop asssets hivercs' do
|
57
|
+
setup do
|
58
|
+
@user_config.stubs(:hivercs).returns(@user_config_hivercs = [mock(), mock()])
|
59
|
+
@hadoop_ext.stubs(:hivercs).returns(@hadoop_ext_hivercs = [mock(), mock(), mock()])
|
60
|
+
end
|
61
|
+
|
62
|
+
should 'apply hiverc options to hive runner call' do
|
63
|
+
@runner_class.expects(:run).with(@user_config_hivercs + @hadoop_ext_hivercs).returns(result = mock())
|
64
|
+
assert_equal result, @command.run([])
|
65
|
+
end
|
66
|
+
|
67
|
+
should 'prepend hiverc options before given params to hive runner call' do
|
68
|
+
params = [mock(), mock()]
|
69
|
+
@runner_class.expects(:run).with(@user_config_hivercs + @hadoop_ext_hivercs + params).returns(result = mock())
|
70
|
+
assert_equal result, @command.run(params)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
context 'without hivercs' do
|
75
|
+
setup do
|
76
|
+
@user_config.stubs(:hivercs).returns([])
|
77
|
+
@hadoop_ext.stubs(:hivercs).returns([])
|
78
|
+
end
|
79
|
+
|
80
|
+
should 'pass params unchanged through to hive runner call' do
|
81
|
+
@runner_class.expects(:run).with(params = [mock(), mock()]).returns(result = mock())
|
82
|
+
assert_equal result, @command.run(params)
|
83
|
+
end
|
84
|
+
|
85
|
+
should 'handle empty params' do
|
86
|
+
@runner_class.expects(:run).with([]).returns(result = mock())
|
87
|
+
assert_equal result, @command.run([])
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
tempdir_context 'running for (mostly) realz' do
|
93
|
+
setup do
|
94
|
+
@conf = ::File.join(@tempdir.path, 'conf')
|
95
|
+
@ext = ::File.join(@tempdir.path, 'hadoop-ext')
|
96
|
+
::Dir.mkdir(@conf)
|
97
|
+
::Dir.mkdir(@ext)
|
98
|
+
@hiverc = @tempdir.file(File.join('conf', 'hiverc')) do |f|
|
99
|
+
f.write(@static_hiverc_content = 'my static content;')
|
100
|
+
f.path
|
101
|
+
end
|
102
|
+
file = Proc.new {|base, name| @tempdir.file(::File.join(base, name)).path }
|
103
|
+
@ext_file = file.call('hadoop-ext', 'a_file.yaml')
|
104
|
+
@ext_jar = file.call('hadoop-ext', 'a_jar.jar')
|
105
|
+
@ext_tar = file.call('hadoop-ext', 'a_tar.tar.gz')
|
106
|
+
@dynamic_hiverc_content = ["ADD FILE #{@ext_file}",
|
107
|
+
"ADD JAR #{@ext_jar}",
|
108
|
+
"ADD ARCHIVE #{@ext_tar}"].join(";\n") + ";\n"
|
109
|
+
@pwd = ::Dir.pwd
|
110
|
+
Hadupils::Search.stubs(:user_config).with.returns(@conf)
|
111
|
+
Hadupils::Runners::Hive.stubs(:base_runner).with.returns(@hive_prog = '/opt/hive/bin/hive')
|
112
|
+
::Dir.chdir @tempdir.path
|
113
|
+
end
|
114
|
+
|
115
|
+
should 'produce a valid set of parameters and hivercs' do
|
116
|
+
Kernel.stubs(:system).with() do |*args|
|
117
|
+
args[0] == @hive_prog and
|
118
|
+
args[1] == '-i' and
|
119
|
+
File.open(args[2], 'r').read == @static_hiverc_content and
|
120
|
+
args[3] == '-i' and
|
121
|
+
File.open(args[4], 'r').read == @dynamic_hiverc_content and
|
122
|
+
args[5] == '--hiveconf' and
|
123
|
+
args[6] == 'my.foo=your.fu'
|
124
|
+
end
|
125
|
+
Hadupils::Commands.run 'hive', ['--hiveconf', 'my.foo=your.fu']
|
126
|
+
end
|
127
|
+
|
128
|
+
teardown do
|
129
|
+
::Dir.chdir @pwd
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
@@ -0,0 +1,266 @@
|
|
1
|
+
class Hadupils::ExtensionsTest < Test::Unit::TestCase
|
2
|
+
context Hadupils::Extensions::Base do
|
3
|
+
context 'initialization with nil path' do
|
4
|
+
should 'have nil as the path' do
|
5
|
+
ext = Hadupils::Extensions::Base.new(nil)
|
6
|
+
assert_equal nil, ext.path
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
context 'initialization' do
|
11
|
+
setup do
|
12
|
+
@path = mock()
|
13
|
+
@expanded_path = mock()
|
14
|
+
@assets = mock()
|
15
|
+
::File.expects(:expand_path).with(@path).returns(@expanded_path)
|
16
|
+
end
|
17
|
+
|
18
|
+
should 'expand the given directory into :path' do
|
19
|
+
Hadupils::Extensions::Base.stubs(:gather_assets).returns(@assets)
|
20
|
+
assert_equal @expanded_path, Hadupils::Extensions::Base.new(@path).path
|
21
|
+
end
|
22
|
+
|
23
|
+
should "gather the expanded directory's assets" do
|
24
|
+
Hadupils::Extensions::Base.expects(:gather_assets).with(@expanded_path).returns(@assets)
|
25
|
+
assert_equal @assets, Hadupils::Extensions::Base.new(@path).assets
|
26
|
+
end
|
27
|
+
|
28
|
+
should "allow manipulation of assets post-expansion" do
|
29
|
+
Hadupils::Extensions::Base.stubs(:gather_assets).returns(@assets)
|
30
|
+
extra = mock()
|
31
|
+
ext = Hadupils::Extensions::Base.new(@path) do
|
32
|
+
assets do |items|
|
33
|
+
# We're just adding the new stuff to the original stuff
|
34
|
+
[items, extra]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
# If the above assets block was applied, we'll see the additional
|
38
|
+
# item there.
|
39
|
+
assert_equal [@assets, extra], ext.assets
|
40
|
+
end
|
41
|
+
|
42
|
+
should 'have an empty hivercs list' do
|
43
|
+
Hadupils::Extensions::Base.stubs(:gather_assets).returns(@assets)
|
44
|
+
assert_equal [], Hadupils::Extensions::Base.new(@path).hivercs
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
context 'gather_assets' do
|
49
|
+
should 'assemble assets with Hadupils::Assets.assets_in' do
|
50
|
+
path = mock()
|
51
|
+
result = mock()
|
52
|
+
Hadupils::Assets.expects(:assets_in).with(path).returns(result)
|
53
|
+
assert_equal result, Hadupils::Extensions::Base.gather_assets(path)
|
54
|
+
end
|
55
|
+
|
56
|
+
should 'allow manipulation of assets' do
|
57
|
+
end
|
58
|
+
|
59
|
+
should 'produce empty list for a nil path' do
|
60
|
+
Hadupils::Assets.expects(:assets_in).never
|
61
|
+
assert_equal [], Hadupils::Extensions::Base.gather_assets(nil)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'a hiverc' do
|
67
|
+
context 'static wrapper' do
|
68
|
+
setup do
|
69
|
+
@klass = Hadupils::Extensions::HiveRC::Static
|
70
|
+
end
|
71
|
+
|
72
|
+
should 'expand the given path into its path attr' do
|
73
|
+
path = 'foo/bar/blah'
|
74
|
+
assert_equal ::File.expand_path(path), @klass.new(path).path
|
75
|
+
end
|
76
|
+
|
77
|
+
should 'provide a close no-op' do
|
78
|
+
assert_respond_to @klass.new('blah'), :close
|
79
|
+
end
|
80
|
+
|
81
|
+
should 'know how to convert to #hive_opts' do
|
82
|
+
path = 'some/awesome/path'
|
83
|
+
assert_equal ['-i', ::File.expand_path(path)],
|
84
|
+
@klass.new(path).hive_opts
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
context 'dynamic wrapper' do
|
89
|
+
setup do
|
90
|
+
@klass = Hadupils::Extensions::HiveRC::Dynamic
|
91
|
+
end
|
92
|
+
|
93
|
+
should 'use Tempfile for its default file_handler' do
|
94
|
+
assert_same ::Tempfile, @klass.file_handler
|
95
|
+
end
|
96
|
+
|
97
|
+
should 'know how to convert to #hive_opts' do
|
98
|
+
obj = @klass.new
|
99
|
+
obj.stubs(:path).returns(path = mock())
|
100
|
+
assert_equal ['-i', obj.path],
|
101
|
+
obj.hive_opts
|
102
|
+
end
|
103
|
+
|
104
|
+
context 'internal file' do
|
105
|
+
setup do
|
106
|
+
@klass.expects(:file_handler).with().returns(@handler = mock())
|
107
|
+
@handler.expects(:new).with('hadupils-hiverc').returns(@file = mock())
|
108
|
+
end
|
109
|
+
|
110
|
+
should "come from the class' file_handler" do
|
111
|
+
assert_equal @file, @klass.new.file
|
112
|
+
end
|
113
|
+
|
114
|
+
should 'provide the path' do
|
115
|
+
@file.stubs(:path).returns(path = mock())
|
116
|
+
::File.stubs(:expand_path).with(path).returns(expanded = mock())
|
117
|
+
assert_equal expanded, @klass.new.path
|
118
|
+
end
|
119
|
+
|
120
|
+
should 'close the file on close()' do
|
121
|
+
@file.expects(:close).with()
|
122
|
+
@klass.new.close
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context 'write operation' do
|
127
|
+
setup do
|
128
|
+
@hiverc = @klass.new
|
129
|
+
@file = File.open(@hiverc.path, 'r')
|
130
|
+
end
|
131
|
+
|
132
|
+
teardown do
|
133
|
+
@file.close
|
134
|
+
@hiverc.close
|
135
|
+
end
|
136
|
+
|
137
|
+
should 'handle simple text lines' do
|
138
|
+
lines = ['some stuff!', 'but what about...', 'this and this!?!']
|
139
|
+
@hiverc.write(lines)
|
140
|
+
expect = lines.join("\n") + "\n"
|
141
|
+
assert_equal expect, @file.read
|
142
|
+
end
|
143
|
+
|
144
|
+
context 'given assets' do
|
145
|
+
setup do
|
146
|
+
@asset_lines = ['ADD FILE foofoo;',
|
147
|
+
'ADD ARCHIVE bloobloo.tar.gz;',
|
148
|
+
'ADD JAR jarjar.jar;']
|
149
|
+
@assets = @asset_lines.collect do |line|
|
150
|
+
m = mock()
|
151
|
+
m.stubs(:hiverc_command).with.returns(line)
|
152
|
+
m
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
should 'use their hiverc_command for lines' do
|
157
|
+
expected = @asset_lines.join("\n") + "\n"
|
158
|
+
@hiverc.write(@assets)
|
159
|
+
assert_equal expected, @file.read
|
160
|
+
end
|
161
|
+
|
162
|
+
should 'handle intermingled text lines' do
|
163
|
+
text_lines = ['some line one', 'some line two']
|
164
|
+
[@assets, @asset_lines].each do |ary|
|
165
|
+
ary.insert(2, text_lines[1])
|
166
|
+
ary.insert(1, text_lines[0])
|
167
|
+
end
|
168
|
+
expected = @asset_lines.join("\n") + "\n"
|
169
|
+
@hiverc.write(@assets)
|
170
|
+
assert_equal expected, @file.read
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
context Hadupils::Extensions::Flat do
|
178
|
+
setup do
|
179
|
+
@klass = Hadupils::Extensions::Flat
|
180
|
+
end
|
181
|
+
|
182
|
+
should 'extend Hadupils::Extensions::Base' do
|
183
|
+
# I usually hate testing this sort of thing, but I want to quickly claim
|
184
|
+
# that @klass has the basic behaviors and focus on what's special about it.
|
185
|
+
assert @klass.ancestors.include? Hadupils::Extensions::Base
|
186
|
+
end
|
187
|
+
|
188
|
+
tempdir_context 'for realz' do
|
189
|
+
setup do
|
190
|
+
@tempdir.file(@file = 'a.file')
|
191
|
+
@tempdir.file(@jar = 'a.jar')
|
192
|
+
@tempdir.file(@archive = 'an.archive.tar.gz')
|
193
|
+
@file_line = "ADD FILE #{@tempdir.full_path(@file)};"
|
194
|
+
@jar_line = "ADD JAR #{@tempdir.full_path(@jar)};"
|
195
|
+
@archive_line = "ADD ARCHIVE #{@tempdir.full_path(@archive)};"
|
196
|
+
end
|
197
|
+
|
198
|
+
should 'produce only one hiverc' do
|
199
|
+
hivercs = @klass.new(@tempdir.path).hivercs
|
200
|
+
assert_equal 1, hivercs.size
|
201
|
+
end
|
202
|
+
|
203
|
+
should 'produce a hiverc for the expected assets' do
|
204
|
+
hivercs = @klass.new(@tempdir.path).hivercs
|
205
|
+
expected = "#{@file_line}\n#{@jar_line}\n#{@archive_line}\n"
|
206
|
+
File.open(hivercs[0].path, 'r') do |f|
|
207
|
+
assert_equal expected, f.read
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
should 'produce a hiverc of a dynamic type' do
|
212
|
+
# This is because I had a bug and was giving the hiverc File
|
213
|
+
# object instead of the dynamic hiverc wrapper object.
|
214
|
+
# Thus it blew up later on.
|
215
|
+
hivercs = @klass.new(@tempdir.path).hivercs
|
216
|
+
assert_kind_of Hadupils::Extensions::HiveRC::Dynamic, hivercs[0]
|
217
|
+
end
|
218
|
+
|
219
|
+
should 'allow manipulation of hiverc items' do
|
220
|
+
extension = @klass.new(@tempdir.path) do
|
221
|
+
hiverc do |assets|
|
222
|
+
assets.insert(1, 'INSERT SOME TEXT HERE')
|
223
|
+
assets << 'FINAL LINE!'
|
224
|
+
end
|
225
|
+
end
|
226
|
+
expected = "#{@file_line}\n" +
|
227
|
+
"INSERT SOME TEXT HERE\n" +
|
228
|
+
"#{@jar_line}\n#{@archive_line}\n" +
|
229
|
+
"FINAL LINE!\n"
|
230
|
+
File.open(extension.hivercs[0].path, 'r') do |f|
|
231
|
+
assert_equal expected, f.read
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
tempdir_context Hadupils::Extensions::Static do
|
238
|
+
setup do
|
239
|
+
@extension = Hadupils::Extensions::Static.new(@tempdir.path)
|
240
|
+
end
|
241
|
+
|
242
|
+
should 'have an empty list of assets from gather_assets' do
|
243
|
+
# These would ordinarily become assets in a dynamic extension.
|
244
|
+
@tempdir.file('some.jar')
|
245
|
+
@tempdir.file('some.tar.gz')
|
246
|
+
@tempdir.file('some.yaml')
|
247
|
+
# but not in this one.
|
248
|
+
assert_equal [], Hadupils::Extensions::Static.new(@tempdir.path).assets
|
249
|
+
end
|
250
|
+
|
251
|
+
should 'have an empty hivercs list when no hiverc file exists' do
|
252
|
+
assert_equal [], @extension.hivercs
|
253
|
+
end
|
254
|
+
|
255
|
+
context 'with a hiverc file' do
|
256
|
+
setup do
|
257
|
+
@hiverc = @tempdir.file('hiverc')
|
258
|
+
end
|
259
|
+
|
260
|
+
should 'have a static HiveRC instance in its hivercs list when a hiverc file exists' do
|
261
|
+
assert_equal [[Hadupils::Extensions::HiveRC::Static, @hiverc.path]],
|
262
|
+
@extension.hivercs.collect {|h| [h.class, h.path] }
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
class Hadupils::RunnersTest < Test::Unit::TestCase
|
2
|
+
context Hadupils::Runners::Base do
|
3
|
+
setup do
|
4
|
+
@runner = Hadupils::Runners::Base.new(@params = mock())
|
5
|
+
end
|
6
|
+
|
7
|
+
should 'expose initialization params as attr' do
|
8
|
+
assert_equal @params, @runner.params
|
9
|
+
end
|
10
|
+
|
11
|
+
context 'wait!' do
|
12
|
+
setup do
|
13
|
+
@command = [mock(), mock(), mock()]
|
14
|
+
@runner.expects(:command).with.returns(@command)
|
15
|
+
# This will ensure that $? is non-nil
|
16
|
+
system(RbConfig.ruby, '-v')
|
17
|
+
end
|
18
|
+
|
19
|
+
should 'assemble system call via command method' do
|
20
|
+
Kernel.expects(:system).with(*@command).returns(true)
|
21
|
+
$?.stubs(:exitstatus).with.returns(mock())
|
22
|
+
@runner.wait!
|
23
|
+
end
|
24
|
+
|
25
|
+
should 'return 255 when system returns nil' do
|
26
|
+
Kernel.stubs(:system).returns(nil)
|
27
|
+
assert_equal 255, @runner.wait!
|
28
|
+
end
|
29
|
+
|
30
|
+
should 'return Process::Status#exitstatus when non-nil system result' do
|
31
|
+
Kernel.stubs(:system).returns(true)
|
32
|
+
$?.stubs(:exitstatus).with.returns(status = mock())
|
33
|
+
assert_equal status, @runner.wait!
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
context Hadupils::Runners::Hive do
|
39
|
+
setup do
|
40
|
+
@klass = Hadupils::Runners::Hive
|
41
|
+
end
|
42
|
+
|
43
|
+
should 'be a runner' do
|
44
|
+
assert_kind_of Hadupils::Runners::Base, @klass.new([])
|
45
|
+
end
|
46
|
+
|
47
|
+
should 'use $HIVE_HOME/bin/hive as the base runner' do
|
48
|
+
ENV.expects(:[]).with('HIVE_HOME').returns(home = mock().to_s)
|
49
|
+
assert_equal ::File.join(home, 'bin', 'hive'),
|
50
|
+
@klass.base_runner
|
51
|
+
end
|
52
|
+
|
53
|
+
context '#command' do
|
54
|
+
setup do
|
55
|
+
@klass.stubs(:base_runner).returns(@hive_path = mock().to_s + '-hive')
|
56
|
+
end
|
57
|
+
|
58
|
+
should 'provide invocation for bare hive if given empty parameters' do
|
59
|
+
assert_equal [@hive_path], @klass.new([]).command
|
60
|
+
end
|
61
|
+
|
62
|
+
should 'provide invocation for hive with all given parameters' do
|
63
|
+
params = [mock().to_s, mock().to_s, mock().to_s, mock().to_s]
|
64
|
+
assert_equal [@hive_path] + params,
|
65
|
+
@klass.new(params).command
|
66
|
+
end
|
67
|
+
|
68
|
+
should 'provide args for hive with :hive_opts on supporting params' do
|
69
|
+
p1 = mock()
|
70
|
+
p1.expects(:hive_opts).with.returns(p1_opts = ['-i', mock().to_s])
|
71
|
+
p2 = mock()
|
72
|
+
p2.expects(:hive_opts).with.returns(p2_opts = ['-i', mock().to_s])
|
73
|
+
s1 = mock().to_s
|
74
|
+
s2 = mock().to_s
|
75
|
+
assert_equal [@hive_path, s1] + p1_opts + [s2] + p2_opts,
|
76
|
+
@klass.new([s1, p1, s2, p2]).command
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
class Hadupils::SearchTest < Test::Unit::TestCase
|
2
|
+
tempdir_context 'find_from_dir' do
|
3
|
+
setup do
|
4
|
+
@module = Hadupils::Search
|
5
|
+
@search_name = mock().to_s
|
6
|
+
end
|
7
|
+
|
8
|
+
should 'return nil if requested directory cannot be found' do
|
9
|
+
assert_equal nil, @module.find_from_dir(@search_name, @tempdir.path)
|
10
|
+
end
|
11
|
+
|
12
|
+
should 'should find the directory when it is in the start dir' do
|
13
|
+
p = @tempdir.full_path('blah')
|
14
|
+
Dir.mkdir p
|
15
|
+
assert_equal p, @module.find_from_dir('blah', @tempdir.path)
|
16
|
+
end
|
17
|
+
|
18
|
+
should 'find the directory when it is in a sibling of the start dir' do
|
19
|
+
target = @tempdir.full_path('target-dir')
|
20
|
+
start = @tempdir.full_path('start-dir')
|
21
|
+
[target, start].each {|d| Dir.mkdir(d) }
|
22
|
+
assert_equal target, @module.find_from_dir('target-dir', start)
|
23
|
+
end
|
24
|
+
|
25
|
+
should 'find the directory when it is above the start dir' do
|
26
|
+
d = @tempdir.full_path('flickityflu')
|
27
|
+
Dir.mkdir(d)
|
28
|
+
assert_equal @tempdir.path,
|
29
|
+
@module.find_from_dir(File.basename(@tempdir.path), d)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'find_from_pwd' do
|
34
|
+
setup do
|
35
|
+
@module = Hadupils::Search
|
36
|
+
@pwd = ::Dir.pwd
|
37
|
+
@target = mock()
|
38
|
+
end
|
39
|
+
|
40
|
+
should 'return the path found by find_from_dir for the pwd' do
|
41
|
+
@module.expects(:find_from_dir).with(@target, @pwd).returns(result = mock())
|
42
|
+
assert_equal result, @module.find_from_pwd(@target)
|
43
|
+
end
|
44
|
+
|
45
|
+
should 'return nil when given that by find_from_dir for the pwd' do
|
46
|
+
@module.expects(:find_from_dir).with(@target, @pwd).returns(nil)
|
47
|
+
assert_equal nil, @module.find_from_pwd(@target)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context 'user_config' do
|
52
|
+
setup do
|
53
|
+
@module = Hadupils::Search
|
54
|
+
end
|
55
|
+
|
56
|
+
should 'use ~/conf by default' do
|
57
|
+
assert_equal ::File.expand_path(::File.join('~', 'conf')),
|
58
|
+
@module.user_config
|
59
|
+
end
|
60
|
+
|
61
|
+
should 'be settable' do
|
62
|
+
assert_equal true, @module.respond_to?(:user_config=)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'hadoop_assets' do
|
67
|
+
should 'search for directory specified by #hadoop_assets_name' do
|
68
|
+
Hadupils::Search.expects(:hadoop_assets_name).with.returns(name = mock().to_s)
|
69
|
+
Hadupils::Search.expects(:find_from_pwd).with(name).returns(dir = mock())
|
70
|
+
assert_equal dir, Hadupils::Search.hadoop_assets
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
context 'hadoop_assets_name' do
|
75
|
+
should 'default to "hadoop-ext"' do
|
76
|
+
assert_equal 'hadoop-ext', Hadupils::Search.hadoop_assets_name
|
77
|
+
end
|
78
|
+
|
79
|
+
should 'be settable' do
|
80
|
+
assert_respond_to Hadupils::Search, :hadoop_assets_name=
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
metadata
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: hadupils
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Ethan Rowe
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-08-15 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: mocha
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: rake
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: should-context
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
description: Provides utilities for dynamic hadoop client environment configuration
|
79
|
+
email: ethan@the-rowes.com
|
80
|
+
executables:
|
81
|
+
- hadupils
|
82
|
+
extensions: []
|
83
|
+
extra_rdoc_files: []
|
84
|
+
files:
|
85
|
+
- lib/hadupils/search.rb
|
86
|
+
- lib/hadupils/commands.rb
|
87
|
+
- lib/hadupils/runners.rb
|
88
|
+
- lib/hadupils/extensions.rb
|
89
|
+
- lib/hadupils/assets.rb
|
90
|
+
- lib/hadupils.rb
|
91
|
+
- test/unit/assets_test.rb
|
92
|
+
- test/unit/commands_test.rb
|
93
|
+
- test/unit/extensions_test.rb
|
94
|
+
- test/unit/runners_test.rb
|
95
|
+
- test/unit/search_test.rb
|
96
|
+
- test/hadupil_test_setup.rb
|
97
|
+
- bin/hadupils
|
98
|
+
- Rakefile.rb
|
99
|
+
- Gemfile.lock
|
100
|
+
- README.md
|
101
|
+
- Gemfile
|
102
|
+
- LICENSE
|
103
|
+
- CHANGELOG.md
|
104
|
+
homepage: http://github.com/ethanrowe/hadupils
|
105
|
+
licenses:
|
106
|
+
- MIT
|
107
|
+
post_install_message:
|
108
|
+
rdoc_options: []
|
109
|
+
require_paths:
|
110
|
+
- lib
|
111
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
112
|
+
none: false
|
113
|
+
requirements:
|
114
|
+
- - ! '>='
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
117
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
118
|
+
none: false
|
119
|
+
requirements:
|
120
|
+
- - ! '>='
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
requirements: []
|
124
|
+
rubyforge_project:
|
125
|
+
rubygems_version: 1.8.25
|
126
|
+
signing_key:
|
127
|
+
specification_version: 3
|
128
|
+
summary: Provides utilities for dynamic hadoop client environment configuration
|
129
|
+
test_files: []
|