snail-map-reduce 0.0.2 → 0.0.3

data/MapReduceLinearMatrixProduct.rb

@@ -0,0 +1,66 @@
+require 'rubygems'
+require 'statsample'
+require './matrix_block_mixin'
+require './map_reduce'
+require 'benchmark'
+
+class Matrix
+	def each_column
+		self.column_vectors.each {|column_vector| yield(column_vector)}
+	end
+	def each_row
+		self.row_vectors.each {|row_vector| yield(row_vector)}
+	end
+end
+
+def pair_up(key, value)
+	inputs = []
+	a = value[:a]
+	b = value[:b]
+
+	row = 0
+	a.each_column do |a_column|
+		column = 0
+		b.row_vectors[row].each do |e|
+			inputs << {:key => column.to_s, :value => {:scalar => e, :vector => a_column}}
+			column += 1
+		end
+		row += 1
+	end
+	inputs
+end
+
+def multiply(key, value)
+	[{ :key => key, :value => value[:vector].collect {|vc| vc * value[:scalar]}}]
+end
+
+def add_reduce(key, values)
+	empty = Array.new(values.first.size)
+	empty.fill(0)
+	v = values.inject(Vector.elements(empty)) {|sum, v| sum + v}
+	[{:key => 'X', :value => v}]
+end
+
+def append_reduce(key, values)
+	[{:key => 'result', :value => Matrix.columns(values)}]
+end
+
+def m(order)
+	Matrix.build(order, order) {|row, col| rand(20) }
+end
+
+order = 8
+mappings = reductions = (Math.log2(order) - 1).to_i
+m1 = m(order)
+m2 = m(order)
+
+operations = []
+operations << Mapper.new {|k,v| pair_up(k,v)}
+operations << Mapper.new {|k,v| multiply(k,v)}
+operations << Reducer.new {|k,vs| add_reduce(k,vs)}
+operations << Reducer.new {|k,vs| append_reduce(k,vs)}
+
+result = MapReduceRunner.new(operations).run([{:key => "X", :value => {:a => m1, :b => m2}}])
+
+puts m1*m2 == result[0][:value]
+

data/MapReduceMatrixProduct.rb

@@ -4,13 +4,13 @@ require './matrix_block_mixin'
 require './map_reduce'
 require 'benchmark'
 
-def setup(key, value)
+def map(key, value)
 	inputs = []
 	a = value[:a]
 	b = value[:b]
 	if a.row_size == 2
 		inputs << {:key=> key, :a => a, :b => b}
-		return
+		return inputs
 	end
 	inputs << {:key => key + "00A", :value => {:a => a.block(0,0), :b => b.block(0,0)}}
 	inputs << {:key => key + "00B", :value => {:a => a.block(0,1), :b => b.block(1,0)}}
@@ -45,40 +45,39 @@ def block_join_reduce(key, values)
 	p01 = values[values.index {|v| v[:identity] == '01'}][:matrix]
 	p10 = values[values.index {|v| v[:identity] == '10'}][:matrix]
 	p11 = values[values.index {|v| v[:identity] == '11'}][:matrix]
-	{:key => key[0..-2], :value => {:identity => key[-1], :matrix => Matrix.rows(join(p00, p01) + join(p10, p11))}}
+	[{:key => key[0..-2], :value => {:identity => key[-1], :matrix => Matrix.rows(join(p00, p01) + join(p10, p11))}}]
 end
 
 def block_matrix_sum(key, values)
 	sum = Matrix.zero(values.first[:matrix].row_size)
 	values.each {|m| sum += m[:matrix]}
-	{:key => key[0..-3], :value => {:matrix => sum, :identity => key[-2..-1]}}
+	[{:key => key[0..-3], :value => {:matrix => sum, :identity => key[-2..-1]}}]
 end
 
 def primitive_map(key, value)
 	[{:key => key[0..-2], :value =>  {:matrix => value[:a] * value[:b], :identity => key[0..-2]}}]
 end
 
-order = 64
+order = 16
 mappings = reductions = (Math.log2(order) - 1).to_i
 m1 = m(order)
 m2 = m(order)
 
-mappers = []
+operations = []
 mappings.times do
-	mappers << ->(k,v) {setup(k,v)}
+	operations << Mapper.new {|k,v| map(k,v)}
 end
 
-mappers << ->(k,v) {primitive_map(k,v)}
-reducers = []
+operations << Mapper.new {|k,v| primitive_map(k,v)}
 
 reductions.times do
-	reducers << ->(k,v) {block_matrix_sum(k,v)}
-	reducers << ->(k,v) {block_join_reduce(k,v)}
+	operations << Reducer.new {|k,v| block_matrix_sum(k,v)}
+	operations << Reducer.new {|k,v| block_join_reduce(k,v)}
 end
 
 result = []
 mr_time = Benchmark.measure do
-	result = MapReduceRunner.new(mappers, reducers).run([{:key => "X", :value => {:a => m1, :b => m2}}])
+	result = MapReduceRunner.new(operations).run([{:key => "X", :value => {:a => m1, :b => m2}}])
 end
 plain_time = Benchmark.measure do
 	m1*m2

data/map_reduce.rb

@@ -11,38 +11,42 @@ class Partitioner
 end
 
 class Reducer
-	def run(partitions)
+	def initialize(&block)
+		@function = block
+	end
+
+	def run(pairs)
+		partitions = Partitioner.new.run(pairs)
 		space = []
 		partitions.each_pair do |k,v|
-			space << yield(k,v)
+			space += @function.call(k,v)
 		end
 		space
 	end
 end
 
 class Mapper
+	def initialize(&block)
+		@function = block
+	end
+
 	def run(pairs)
 		mapped_pairs = []
 		pairs.each do |pair|
-			mapped_pairs += yield(pair[:key], pair[:value])
+			mapped_pairs += @function.call(pair[:key], pair[:value])
 		end
 		mapped_pairs
 	end
 end
 
 class MapReduceRunner
-	def initialize(mappers, reducers)
-		@mappers = mappers
-		@reducers = reducers
+	def initialize(operations)
+		@operations = operations
 	end
 	
 	def run(pairs)
 		results = []
-		@mappers.each {|mapper| pairs = Mapper.new.run(pairs) {|k,v| mapper.call(k,v)}}
-		@reducers.each do |reducer|
-			partitions = Partitioner.new.run(pairs)
-			pairs = Reducer.new.run(partitions) {|k,v| reducer.call(k,v)}
-		end
+		@operations.each {|operation| pairs = operation.run(pairs)}
 		pairs
 	end
 end

data/snail.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s| 
   s.name = "snail-map-reduce"
-  s.version = "0.0.2"
+  s.version = "0.0.3"
   s.author = "Avishek Sen Gupta"
   s.email = "avishek.sen.gupta@gmail.com"
   s.homepage = "http://avishek.net/blog"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: snail-map-reduce
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -19,6 +19,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- MapReduceLinearMatrixProduct.rb
 - MapReduceMatrixProduct.rb
 - README
 - map_reduce.rb