hadoop-jruby-connector 0.0.6 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/hjc/hadoop_streaming.rb +1 -8
- data/spec/hjc/hadoop_streaming_spec.rb +2 -10
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.7
|
data/lib/hjc/hadoop_streaming.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Hjc
|
2
2
|
class HadoopStreaming
|
3
|
-
attr_accessor :input_path, :output_path, :mapper_path, :reducer_path, :jobconf
|
3
|
+
attr_accessor :input_path, :output_path, :mapper_path, :reducer_path, :jobconf
|
4
4
|
attr_accessor :local, :debug
|
5
5
|
|
6
6
|
def initialize
|
@@ -30,13 +30,11 @@ module Hjc
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def mapper=(mapper)
|
33
|
-
mapper = with_shebang(mapper, @language)
|
34
33
|
@files['mapper'] = file = Util.to_temp_file('mapper', mapper, :mod => 0700)
|
35
34
|
@mapper_path = File.basename(file.path)
|
36
35
|
end
|
37
36
|
|
38
37
|
def reducer=(reducer)
|
39
|
-
reducer = with_shebang(reducer, @language)
|
40
38
|
@files['reducer'] = file = Util.to_temp_file('reducer', reducer, :mod => 0700)
|
41
39
|
@reducer_path = File.basename(file.path)
|
42
40
|
end
|
@@ -62,10 +60,5 @@ module Hjc
|
|
62
60
|
puts "args: #{concated_args.join(' ')}" if @debug
|
63
61
|
concated_args
|
64
62
|
end
|
65
|
-
|
66
|
-
private
|
67
|
-
def with_shebang(body, language)
|
68
|
-
"#!/usr/bin/env #{language}\n#{body}"
|
69
|
-
end
|
70
63
|
end
|
71
64
|
end
|
@@ -34,16 +34,6 @@ module Hjc
|
|
34
34
|
job.args.join(" ").should match(/reducer/)
|
35
35
|
end
|
36
36
|
|
37
|
-
it 'adds shebang if language exists' do
|
38
|
-
job = HadoopStreaming.new
|
39
|
-
job.language = 'ruby'
|
40
|
-
job.mapper = 'mapper'
|
41
|
-
job.reducer = 'reducer'
|
42
|
-
|
43
|
-
File.read(File.join(TMP_DIR, job.mapper_path)).should match(%Q:#!/usr/bin/env ruby:)
|
44
|
-
File.read(File.join(TMP_DIR, job.reducer_path)).should match(%Q:#!/usr/bin/env ruby:)
|
45
|
-
end
|
46
|
-
|
47
37
|
it 'can run Hadoop streaming job with path' do
|
48
38
|
pending 'path does not work'
|
49
39
|
job = HadoopStreaming.new
|
@@ -101,6 +91,7 @@ module Hjc
|
|
101
91
|
end
|
102
92
|
|
103
93
|
MAPPER = <<-'EOF'
|
94
|
+
#!/usr/bin/env ruby
|
104
95
|
ARGF.each do |line|
|
105
96
|
line.chomp!
|
106
97
|
line.split.each do |word|
|
@@ -110,6 +101,7 @@ end
|
|
110
101
|
EOF
|
111
102
|
|
112
103
|
REDUCER = <<-'EOF'
|
104
|
+
#!/usr/bin/env ruby
|
113
105
|
count = Hash.new {|h,k| h[k] = 0}
|
114
106
|
ARGF.each do |line|
|
115
107
|
line.chomp!
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 7
|
9
|
+
version: 0.0.7
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Koichi Fujikawa
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2011-
|
17
|
+
date: 2011-06-06 00:00:00 +09:00
|
18
18
|
default_executable:
|
19
19
|
dependencies: []
|
20
20
|
|