snail-map-reduce 0.0.1

data/.gitignore

@@ -0,0 +1,10 @@
+*.*~
+artifacts
+csv
+para
+som
+*.aux
+*.dvi
+*.log
+*.swp
+*~

data/MapReduceMatrixProduct.rb

@@ -0,0 +1,85 @@
+require 'rubygems'
+require 'statsample'
+require './matrix_block_mixin'
+require './map_reduce'
+
+class Inputs
+	attr_accessor :inputs
+	def initialize
+		@inputs = []
+	end
+
+	def setup(a,b,key)
+		if a.row_size == 2
+			@inputs << {:key=> key, :a => a, :b => b}
+			return
+		end
+		setup(a.block(0,0), b.block(0,0), key + "00A")
+		setup(a.block(0,1), b.block(1,0), key + "00B")
+
+		setup(a.block(0,0), b.block(0,1), key + "01A")
+		setup(a.block(0,1), b.block(1,1), key + "01B")
+
+		setup(a.block(1,0), b.block(0,0), key + "10A")
+		setup(a.block(1,1), b.block(1,0), key + "10B")
+
+		setup(a.block(1,0), b.block(0,1), key + "11A")
+		setup(a.block(1,1), b.block(1,1), key + "11B")
+	
+	end
+end
+
+def join(left_block, right_block)
+	rows = []
+	lower_order = left_block.row_size
+	lower_order.times do |t|
+		rows << left_block.row(t).to_a + right_block.row(t).to_a
+	end
+	rows
+end
+
+def m(order)
+	Matrix.build(order, order) {|row, col| rand(20) }
+end
+
+def block_join_reduce(key, values)
+	p00 = values[values.index {|v| v[:identity] == '00'}][:matrix]
+	p01 = values[values.index {|v| v[:identity] == '01'}][:matrix]
+	p10 = values[values.index {|v| v[:identity] == '10'}][:matrix]
+	p11 = values[values.index {|v| v[:identity] == '11'}][:matrix]
+	{:key => key[0..-2], :value => {:identity => key[-1], :matrix => Matrix.rows(join(p00, p01) + join(p10, p11))}}
+end
+
+def block_matrix_sum(key, values)
+	sum = Matrix.zero(values.first[:matrix].row_size)
+	values.each {|m| sum += m[:matrix]}
+	{:key => key[0..-3], :value => {:matrix => sum, :identity => key[-2..-1]}}
+end
+
+def primitive_map(key, value)
+	{:key => key[0..-2], :value =>  {:matrix => value[:a] * value[:b], :identity => key[0..-2]}}
+end
+
+order = 32
+reductions = (Math.log2(order) - 1).to_i
+m1 = m(order)
+m2 = m(order)
+
+inputs = Inputs.new
+inputs.setup(m1,m2,"X")
+space = inputs.inputs
+
+
+mappers = [->(k,v) {primitive_map(k,v)}]
+reducers = []
+
+reductions.times do
+	reducers << ->(k,v) {block_matrix_sum(k,v)}
+	reducers << ->(k,v) {block_join_reduce(k,v)}
+end
+
+result = MapReduceRunner.new(mappers, reducers).run(space)
+puts result
+puts result[0][:value][:matrix]
+puts m1*m2 == result[0][:value][:matrix]
+

data/README

@@ -0,0 +1,2 @@
+Snail is a single-threaded, in-memory, barebones MapReduce framework written in Ruby to quickly prototype and test parallel algorithms. It's so called because all its operations are in a single-thread, so it runs really slow for even medium size datasets. However, Snail is intended to help prototype the logic and verify the correctness of a MapReduce algorithm, thus speed/scaling are not the aims.
+

data/map_reduce.rb

@@ -0,0 +1,45 @@
+class Partitioner
+	def run(space)
+		partitions = {}
+		space.each do |i|
+			key = i[:key]
+			partitions[key] = [] if partitions[key].nil?
+			partitions[key] << i[:value]
+		end
+		partitions
+	end
+end
+
+class Reducer
+	def run(partitions)
+		space = []
+		partitions.each_pair do |k,v|
+			space << yield(k,v)
+		end
+		space
+	end
+end
+
+class Mapper
+	def run(space)
+		space.collect {|i| yield(i[:key], i)}
+	end
+end
+
+class MapReduceRunner
+	def initialize(mappers, reducers)
+		@mappers = mappers
+		@reducers = reducers
+	end
+	
+	def run(space)
+		results = []
+		@mappers.each {|mapper| results = Mapper.new.run(space) {|k,v| mapper.call(k,v)}}
+		@reducers.each do |reducer|
+			partitions = Partitioner.new.run(results)
+			results = Reducer.new.run(partitions) {|k,v| reducer.call(k,v)}
+		end
+		results
+	end
+end
+

data/matrix_block_mixin.rb

@@ -0,0 +1,18 @@
+class Matrix
+	def block(block_row, block_column)
+		raise "Non 2^n matrix" if (row_size & (row_size - 1)) != 0 || (column_size & (column_size - 1)) != 0
+		lower_order = row_size/2
+		start_row = block_row * lower_order
+		start_column = block_column * lower_order
+		b = []
+		lower_order.times do |r|
+			row = []
+			lower_order.times do |c|
+				row << self[start_row + r, start_column + c]
+			end
+			b << row
+		end
+		Matrix.rows(b)
+	end
+end
+

data/snail.gemspec

@@ -0,0 +1,13 @@
+spec = Gem::Specification.new do |s| 
+  s.name = "snail-map-reduce"
+  s.version = "0.0.1"
+  s.author = "Avishek Sen Gupta"
+  s.email = "avishek.sen.gupta@gmail.com"
+  s.homepage = "http://avishek.net/blog"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "Some description"
+  s.files = `git ls-files`.split("\n")
+  s.summary = %q{Snail is a single-threaded, in-memory, barebones MapReduce framework written in Ruby to quickly prototype and test parallel algorithms.}
+  s.description = %q{Snail is a single-threaded, in-memory, barebones MapReduce framework written in Ruby to quickly prototype and test parallel algorithms.}
+end
+

metadata

@@ -0,0 +1,52 @@
+--- !ruby/object:Gem::Specification
+name: snail-map-reduce
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Avishek Sen Gupta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-11-10 00:00:00.000000000Z
+dependencies: []
+description: Snail is a single-threaded, in-memory, barebones MapReduce framework
+  written in Ruby to quickly prototype and test parallel algorithms.
+email: avishek.sen.gupta@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- MapReduceMatrixProduct.rb
+- README
+- map_reduce.rb
+- matrix_block_mixin.rb
+- snail.gemspec
+homepage: http://avishek.net/blog
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Snail is a single-threaded, in-memory, barebones MapReduce framework written
+  in Ruby to quickly prototype and test parallel algorithms.
+test_files: []