hadoop-jruby-connector 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/lib/hjc/hadoop_streaming.rb +2 -2
- data/lib/hjc/util.rb +29 -23
- data/spec/hjc/util_spec.rb +7 -0
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.6
|
data/lib/hjc/hadoop_streaming.rb
CHANGED
@@ -22,10 +22,10 @@ module Hjc
|
|
22
22
|
def input=(input)
|
23
23
|
# input param seems to explain exact path on Hadoop streaming..
|
24
24
|
file = Util.to_temp_file('input', input)
|
25
|
-
@input_path = file
|
25
|
+
@input_path = Util.rel_path(file)
|
26
26
|
unless @local # path seems on HDFS
|
27
27
|
sh = FsShell.new
|
28
|
-
sh.put(file.path, file
|
28
|
+
sh.put(file.path, Util.rel_path(file))
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
data/lib/hjc/util.rb
CHANGED
@@ -4,38 +4,44 @@ module Hjc
|
|
4
4
|
class Util
|
5
5
|
TMP_DIR = 'tmp'
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
|
7
|
+
class << self
|
8
|
+
def setup
|
9
|
+
hadoop_jars.each do |jar|
|
10
|
+
require jar # fro JRuby
|
11
|
+
end
|
12
|
+
|
13
|
+
$CLASSPATH << "#{hadoop_home}/conf"
|
10
14
|
end
|
11
|
-
|
12
|
-
$CLASSPATH << "#{hadoop_home}/conf"
|
13
|
-
end
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
16
|
+
def hadoop_jars
|
17
|
+
jars = []
|
18
|
+
['', 'lib', 'contrib/streaming'].each do |path|
|
19
|
+
jars.concat Dir.glob(File.join(hadoop_home, path, "*.jar"))
|
20
|
+
end
|
21
|
+
jars
|
19
22
|
end
|
20
|
-
jars
|
21
|
-
end
|
22
23
|
|
23
|
-
|
24
|
-
|
25
|
-
|
24
|
+
def hadoop_home
|
25
|
+
ENV['HADOOP_HOME']
|
26
|
+
end
|
26
27
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
def to_temp_file(filename, body, options={})
|
29
|
+
file = Tempfile.new(filename, TMP_DIR)
|
30
|
+
file.print body
|
31
|
+
file.close
|
31
32
|
|
32
|
-
|
33
|
-
|
34
|
-
|
33
|
+
unless options.empty?
|
34
|
+
if options.keys.include? :mod
|
35
|
+
File.chmod(options[:mod], file.path)
|
36
|
+
end
|
35
37
|
end
|
38
|
+
|
39
|
+
file
|
36
40
|
end
|
37
41
|
|
38
|
-
file
|
42
|
+
def rel_path(file)
|
43
|
+
File.join(TMP_DIR, File.basename(file.path))
|
44
|
+
end
|
39
45
|
end
|
40
46
|
end
|
41
47
|
end
|
data/spec/hjc/util_spec.rb
CHANGED
@@ -14,5 +14,12 @@ module Hjc
|
|
14
14
|
|
15
15
|
FileTest.executable?(f.path).should be_true
|
16
16
|
end
|
17
|
+
|
18
|
+
it 'returns relative path from tempfile' do
|
19
|
+
f = Util.to_temp_file('map.rb', 'mapscript', :mod => 0700)
|
20
|
+
|
21
|
+
filename = File.basename(f.path)
|
22
|
+
Util.rel_path(f).should == "tmp/#{filename}"
|
23
|
+
end
|
17
24
|
end
|
18
25
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 6
|
9
|
+
version: 0.0.6
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Koichi Fujikawa
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2011-05-
|
17
|
+
date: 2011-05-30 00:00:00 +09:00
|
18
18
|
default_executable:
|
19
19
|
dependencies: []
|
20
20
|
|