hadoop-jruby-connector 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/hjc/hadoop_streaming.rb +2 -2
- data/lib/hjc/util.rb +29 -23
- data/spec/hjc/util_spec.rb +7 -0
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.6
|
data/lib/hjc/hadoop_streaming.rb
CHANGED
@@ -22,10 +22,10 @@ module Hjc
|
|
22
22
|
def input=(input)
|
23
23
|
# input param seems to explain exact path on Hadoop streaming..
|
24
24
|
file = Util.to_temp_file('input', input)
|
25
|
-
@input_path = file
|
25
|
+
@input_path = Util.rel_path(file)
|
26
26
|
unless @local # path seems on HDFS
|
27
27
|
sh = FsShell.new
|
28
|
-
sh.put(file.path, file
|
28
|
+
sh.put(file.path, Util.rel_path(file))
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
data/lib/hjc/util.rb
CHANGED
@@ -4,38 +4,44 @@ module Hjc
|
|
4
4
|
class Util
|
5
5
|
TMP_DIR = 'tmp'
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
|
7
|
+
class << self
|
8
|
+
def setup
|
9
|
+
hadoop_jars.each do |jar|
|
10
|
+
require jar # fro JRuby
|
11
|
+
end
|
12
|
+
|
13
|
+
$CLASSPATH << "#{hadoop_home}/conf"
|
10
14
|
end
|
11
|
-
|
12
|
-
$CLASSPATH << "#{hadoop_home}/conf"
|
13
|
-
end
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
16
|
+
def hadoop_jars
|
17
|
+
jars = []
|
18
|
+
['', 'lib', 'contrib/streaming'].each do |path|
|
19
|
+
jars.concat Dir.glob(File.join(hadoop_home, path, "*.jar"))
|
20
|
+
end
|
21
|
+
jars
|
19
22
|
end
|
20
|
-
jars
|
21
|
-
end
|
22
23
|
|
23
|
-
|
24
|
-
|
25
|
-
|
24
|
+
def hadoop_home
|
25
|
+
ENV['HADOOP_HOME']
|
26
|
+
end
|
26
27
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
def to_temp_file(filename, body, options={})
|
29
|
+
file = Tempfile.new(filename, TMP_DIR)
|
30
|
+
file.print body
|
31
|
+
file.close
|
31
32
|
|
32
|
-
|
33
|
-
|
34
|
-
|
33
|
+
unless options.empty?
|
34
|
+
if options.keys.include? :mod
|
35
|
+
File.chmod(options[:mod], file.path)
|
36
|
+
end
|
35
37
|
end
|
38
|
+
|
39
|
+
file
|
36
40
|
end
|
37
41
|
|
38
|
-
file
|
42
|
+
def rel_path(file)
|
43
|
+
File.join(TMP_DIR, File.basename(file.path))
|
44
|
+
end
|
39
45
|
end
|
40
46
|
end
|
41
47
|
end
|
data/spec/hjc/util_spec.rb
CHANGED
@@ -14,5 +14,12 @@ module Hjc
|
|
14
14
|
|
15
15
|
FileTest.executable?(f.path).should be_true
|
16
16
|
end
|
17
|
+
|
18
|
+
it 'returns relative path from tempfile' do
|
19
|
+
f = Util.to_temp_file('map.rb', 'mapscript', :mod => 0700)
|
20
|
+
|
21
|
+
filename = File.basename(f.path)
|
22
|
+
Util.rel_path(f).should == "tmp/#{filename}"
|
23
|
+
end
|
17
24
|
end
|
18
25
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 6
|
9
|
+
version: 0.0.6
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Koichi Fujikawa
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2011-05-
|
17
|
+
date: 2011-05-30 00:00:00 +09:00
|
18
18
|
default_executable:
|
19
19
|
dependencies: []
|
20
20
|
|