mandy 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. data/bin/mandy-local +2 -1
  2. data/lib/job.rb +11 -4
  3. data/lib/mandy.rb +2 -1
  4. metadata +2 -2
data/bin/mandy-local CHANGED
@@ -47,7 +47,8 @@ out = nil
47
47
  Mandy::Job.jobs.each_with_index do |job, i|
48
48
  out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
49
49
  puts "Running #{job.name}..."
50
- `cat #{input} | mandy-map #{file} "#{job.name}" | sort | mandy-reduce #{file} "#{job.name}" > #{out}`
50
+ reduce_phase = job.reducer_defined? ? %(| sort | mandy-reduce #{file} "#{job.name}") : ''
51
+ `cat #{input} | mandy-map #{file} "#{job.name}" #{reduce_phase} > #{out}`
51
52
  input = out
52
53
  end
53
54
 
data/lib/job.rb CHANGED
@@ -18,8 +18,10 @@ module Mandy
18
18
  @name = name
19
19
  @settings = {}
20
20
  @modules = []
21
+ @map, @reduce = nil, nil
21
22
  set('mapred.job.name', name)
22
23
  instance_eval(&blk) if blk
24
+ auto_set_reduce_count
23
25
  end
24
26
 
25
27
  def mixin(*modules)
@@ -88,7 +90,16 @@ module Mandy
88
90
  reducer.execute
89
91
  end
90
92
 
93
+ def reducer_defined?
94
+ !@reduce.nil?
95
+ end
96
+
91
97
  private
98
+
99
+ def auto_set_reduce_count
100
+ return if settings.has_key?('mapred.reduce.tasks')
101
+ reduce_tasks(reducer_defined? ? 1 : 0)
102
+ end
92
103
 
93
104
  def mapper_class
94
105
  return Mandy::Mappers::PassThroughMapper unless @map
@@ -117,10 +128,6 @@ module Mandy
117
128
  @modules.each {|m| @reducer_class.send(:include, m) }
118
129
  @reducer_class
119
130
  end
120
-
121
- def reducer_defined?
122
- reducer_class != Mandy::Reducers::PassThroughReducer
123
- end
124
131
 
125
132
  end
126
133
  end
data/lib/mandy.rb CHANGED
@@ -65,7 +65,8 @@ at_exit do
65
65
  Mandy::Job.jobs.each_with_index do |job, i|
66
66
  out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
67
67
  puts "Running #{job.name}..."
68
- `cat #{input} | mandy-map #{file} "#{job.name}" | sort | mandy-reduce #{file} "#{job.name}" > #{out}`
68
+ reduce_phase = job.reducer_defined? ? %(| sort | mandy-reduce #{file} "#{job.name}") : ''
69
+ `cat #{input} | mandy-map #{file} "#{job.name}" #{reduce_phase} > #{out}`
69
70
  input = out
70
71
  end
71
72
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mandy
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.3
4
+ version: 0.4.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andy Kent
@@ -10,7 +10,7 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2009-10-05 00:00:00 +01:00
13
+ date: 2009-10-15 00:00:00 +01:00
14
14
  default_executable:
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency