mandy 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/mandy-local +2 -1
- data/lib/job.rb +11 -4
- data/lib/mandy.rb +2 -1
- metadata +2 -2
data/bin/mandy-local
CHANGED
|
@@ -47,7 +47,8 @@ out = nil
|
|
|
47
47
|
Mandy::Job.jobs.each_with_index do |job, i|
|
|
48
48
|
out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
|
|
49
49
|
puts "Running #{job.name}..."
|
|
50
|
-
|
|
50
|
+
reduce_phase = job.reducer_defined? ? %(| sort | mandy-reduce #{file} "#{job.name}") : ''
|
|
51
|
+
`cat #{input} | mandy-map #{file} "#{job.name}" #{reduce_phase} > #{out}`
|
|
51
52
|
input = out
|
|
52
53
|
end
|
|
53
54
|
|
data/lib/job.rb
CHANGED
|
@@ -18,8 +18,10 @@ module Mandy
|
|
|
18
18
|
@name = name
|
|
19
19
|
@settings = {}
|
|
20
20
|
@modules = []
|
|
21
|
+
@map, @reduce = nil, nil
|
|
21
22
|
set('mapred.job.name', name)
|
|
22
23
|
instance_eval(&blk) if blk
|
|
24
|
+
auto_set_reduce_count
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
def mixin(*modules)
|
|
@@ -88,7 +90,16 @@ module Mandy
|
|
|
88
90
|
reducer.execute
|
|
89
91
|
end
|
|
90
92
|
|
|
93
|
+
def reducer_defined?
|
|
94
|
+
!@reduce.nil?
|
|
95
|
+
end
|
|
96
|
+
|
|
91
97
|
private
|
|
98
|
+
|
|
99
|
+
def auto_set_reduce_count
|
|
100
|
+
return if settings.has_key?('mapred.reduce.tasks')
|
|
101
|
+
reduce_tasks(reducer_defined? ? 1 : 0)
|
|
102
|
+
end
|
|
92
103
|
|
|
93
104
|
def mapper_class
|
|
94
105
|
return Mandy::Mappers::PassThroughMapper unless @map
|
|
@@ -117,10 +128,6 @@ module Mandy
|
|
|
117
128
|
@modules.each {|m| @reducer_class.send(:include, m) }
|
|
118
129
|
@reducer_class
|
|
119
130
|
end
|
|
120
|
-
|
|
121
|
-
def reducer_defined?
|
|
122
|
-
reducer_class != Mandy::Reducers::PassThroughReducer
|
|
123
|
-
end
|
|
124
131
|
|
|
125
132
|
end
|
|
126
133
|
end
|
data/lib/mandy.rb
CHANGED
|
@@ -65,7 +65,8 @@ at_exit do
|
|
|
65
65
|
Mandy::Job.jobs.each_with_index do |job, i|
|
|
66
66
|
out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
|
|
67
67
|
puts "Running #{job.name}..."
|
|
68
|
-
|
|
68
|
+
reduce_phase = job.reducer_defined? ? %(| sort | mandy-reduce #{file} "#{job.name}") : ''
|
|
69
|
+
`cat #{input} | mandy-map #{file} "#{job.name}" #{reduce_phase} > #{out}`
|
|
69
70
|
input = out
|
|
70
71
|
end
|
|
71
72
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mandy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andy Kent
|
|
@@ -10,7 +10,7 @@ autorequire:
|
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
12
|
|
|
13
|
-
date: 2009-10-
|
|
13
|
+
date: 2009-10-15 00:00:00 +01:00
|
|
14
14
|
default_executable:
|
|
15
15
|
dependencies:
|
|
16
16
|
- !ruby/object:Gem::Dependency
|