snail-map-reduce 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/MapReduceLinearMatrixProduct.rb +66 -0
- data/MapReduceMatrixProduct.rb +11 -12
- data/map_reduce.rb +15 -11
- data/snail.gemspec +1 -1
- metadata +2 -1
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'statsample'
|
3
|
+
require './matrix_block_mixin'
|
4
|
+
require './map_reduce'
|
5
|
+
require 'benchmark'
|
6
|
+
|
7
|
+
class Matrix
|
8
|
+
def each_column
|
9
|
+
self.column_vectors.each {|column_vector| yield(column_vector)}
|
10
|
+
end
|
11
|
+
def each_row
|
12
|
+
self.row_vectors.each {|row_vector| yield(row_vector)}
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def pair_up(key, value)
|
17
|
+
inputs = []
|
18
|
+
a = value[:a]
|
19
|
+
b = value[:b]
|
20
|
+
|
21
|
+
row = 0
|
22
|
+
a.each_column do |a_column|
|
23
|
+
column = 0
|
24
|
+
b.row_vectors[row].each do |e|
|
25
|
+
inputs << {:key => column.to_s, :value => {:scalar => e, :vector => a_column}}
|
26
|
+
column += 1
|
27
|
+
end
|
28
|
+
row += 1
|
29
|
+
end
|
30
|
+
inputs
|
31
|
+
end
|
32
|
+
|
33
|
+
def multiply(key, value)
|
34
|
+
[{ :key => key, :value => value[:vector].collect {|vc| vc * value[:scalar]}}]
|
35
|
+
end
|
36
|
+
|
37
|
+
def add_reduce(key, values)
|
38
|
+
empty = Array.new(values.first.size)
|
39
|
+
empty.fill(0)
|
40
|
+
v = values.inject(Vector.elements(empty)) {|sum, v| sum + v}
|
41
|
+
[{:key => 'X', :value => v}]
|
42
|
+
end
|
43
|
+
|
44
|
+
def append_reduce(key, values)
|
45
|
+
[{:key => 'result', :value => Matrix.columns(values)}]
|
46
|
+
end
|
47
|
+
|
48
|
+
def m(order)
|
49
|
+
Matrix.build(order, order) {|row, col| rand(20) }
|
50
|
+
end
|
51
|
+
|
52
|
+
order = 8
|
53
|
+
mappings = reductions = (Math.log2(order) - 1).to_i
|
54
|
+
m1 = m(order)
|
55
|
+
m2 = m(order)
|
56
|
+
|
57
|
+
operations = []
|
58
|
+
operations << Mapper.new {|k,v| pair_up(k,v)}
|
59
|
+
operations << Mapper.new {|k,v| multiply(k,v)}
|
60
|
+
operations << Reducer.new {|k,vs| add_reduce(k,vs)}
|
61
|
+
operations << Reducer.new {|k,vs| append_reduce(k,vs)}
|
62
|
+
|
63
|
+
result = MapReduceRunner.new(operations).run([{:key => "X", :value => {:a => m1, :b => m2}}])
|
64
|
+
|
65
|
+
puts m1*m2 == result[0][:value]
|
66
|
+
|
data/MapReduceMatrixProduct.rb
CHANGED
@@ -4,13 +4,13 @@ require './matrix_block_mixin'
|
|
4
4
|
require './map_reduce'
|
5
5
|
require 'benchmark'
|
6
6
|
|
7
|
-
def
|
7
|
+
def map(key, value)
|
8
8
|
inputs = []
|
9
9
|
a = value[:a]
|
10
10
|
b = value[:b]
|
11
11
|
if a.row_size == 2
|
12
12
|
inputs << {:key=> key, :a => a, :b => b}
|
13
|
-
return
|
13
|
+
return inputs
|
14
14
|
end
|
15
15
|
inputs << {:key => key + "00A", :value => {:a => a.block(0,0), :b => b.block(0,0)}}
|
16
16
|
inputs << {:key => key + "00B", :value => {:a => a.block(0,1), :b => b.block(1,0)}}
|
@@ -45,40 +45,39 @@ def block_join_reduce(key, values)
|
|
45
45
|
p01 = values[values.index {|v| v[:identity] == '01'}][:matrix]
|
46
46
|
p10 = values[values.index {|v| v[:identity] == '10'}][:matrix]
|
47
47
|
p11 = values[values.index {|v| v[:identity] == '11'}][:matrix]
|
48
|
-
{:key => key[0..-2], :value => {:identity => key[-1], :matrix => Matrix.rows(join(p00, p01) + join(p10, p11))}}
|
48
|
+
[{:key => key[0..-2], :value => {:identity => key[-1], :matrix => Matrix.rows(join(p00, p01) + join(p10, p11))}}]
|
49
49
|
end
|
50
50
|
|
51
51
|
def block_matrix_sum(key, values)
|
52
52
|
sum = Matrix.zero(values.first[:matrix].row_size)
|
53
53
|
values.each {|m| sum += m[:matrix]}
|
54
|
-
{:key => key[0..-3], :value => {:matrix => sum, :identity => key[-2..-1]}}
|
54
|
+
[{:key => key[0..-3], :value => {:matrix => sum, :identity => key[-2..-1]}}]
|
55
55
|
end
|
56
56
|
|
57
57
|
def primitive_map(key, value)
|
58
58
|
[{:key => key[0..-2], :value => {:matrix => value[:a] * value[:b], :identity => key[0..-2]}}]
|
59
59
|
end
|
60
60
|
|
61
|
-
order =
|
61
|
+
order = 16
|
62
62
|
mappings = reductions = (Math.log2(order) - 1).to_i
|
63
63
|
m1 = m(order)
|
64
64
|
m2 = m(order)
|
65
65
|
|
66
|
-
|
66
|
+
operations = []
|
67
67
|
mappings.times do
|
68
|
-
|
68
|
+
operations << Mapper.new {|k,v| map(k,v)}
|
69
69
|
end
|
70
70
|
|
71
|
-
|
72
|
-
reducers = []
|
71
|
+
operations << Mapper.new {|k,v| primitive_map(k,v)}
|
73
72
|
|
74
73
|
reductions.times do
|
75
|
-
|
76
|
-
|
74
|
+
operations << Reducer.new {|k,v| block_matrix_sum(k,v)}
|
75
|
+
operations << Reducer.new {|k,v| block_join_reduce(k,v)}
|
77
76
|
end
|
78
77
|
|
79
78
|
result = []
|
80
79
|
mr_time = Benchmark.measure do
|
81
|
-
result = MapReduceRunner.new(
|
80
|
+
result = MapReduceRunner.new(operations).run([{:key => "X", :value => {:a => m1, :b => m2}}])
|
82
81
|
end
|
83
82
|
plain_time = Benchmark.measure do
|
84
83
|
m1*m2
|
data/map_reduce.rb
CHANGED
@@ -11,38 +11,42 @@ class Partitioner
|
|
11
11
|
end
|
12
12
|
|
13
13
|
class Reducer
|
14
|
-
def
|
14
|
+
def initialize(&block)
|
15
|
+
@function = block
|
16
|
+
end
|
17
|
+
|
18
|
+
def run(pairs)
|
19
|
+
partitions = Partitioner.new.run(pairs)
|
15
20
|
space = []
|
16
21
|
partitions.each_pair do |k,v|
|
17
|
-
space
|
22
|
+
space += @function.call(k,v)
|
18
23
|
end
|
19
24
|
space
|
20
25
|
end
|
21
26
|
end
|
22
27
|
|
23
28
|
class Mapper
|
29
|
+
def initialize(&block)
|
30
|
+
@function = block
|
31
|
+
end
|
32
|
+
|
24
33
|
def run(pairs)
|
25
34
|
mapped_pairs = []
|
26
35
|
pairs.each do |pair|
|
27
|
-
mapped_pairs +=
|
36
|
+
mapped_pairs += @function.call(pair[:key], pair[:value])
|
28
37
|
end
|
29
38
|
mapped_pairs
|
30
39
|
end
|
31
40
|
end
|
32
41
|
|
33
42
|
class MapReduceRunner
|
34
|
-
def initialize(
|
35
|
-
@
|
36
|
-
@reducers = reducers
|
43
|
+
def initialize(operations)
|
44
|
+
@operations = operations
|
37
45
|
end
|
38
46
|
|
39
47
|
def run(pairs)
|
40
48
|
results = []
|
41
|
-
@
|
42
|
-
@reducers.each do |reducer|
|
43
|
-
partitions = Partitioner.new.run(pairs)
|
44
|
-
pairs = Reducer.new.run(partitions) {|k,v| reducer.call(k,v)}
|
45
|
-
end
|
49
|
+
@operations.each {|operation| pairs = operation.run(pairs)}
|
46
50
|
pairs
|
47
51
|
end
|
48
52
|
end
|
data/snail.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: snail-map-reduce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -19,6 +19,7 @@ extensions: []
|
|
19
19
|
extra_rdoc_files: []
|
20
20
|
files:
|
21
21
|
- .gitignore
|
22
|
+
- MapReduceLinearMatrixProduct.rb
|
22
23
|
- MapReduceMatrixProduct.rb
|
23
24
|
- README
|
24
25
|
- map_reduce.rb
|