map_reduce 0.0.1.alpha

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in map_reduce.gemspec
+gemspec
+
+group :test do
+  gem "minitest"
+  gem "minitest-reporters"
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Petr Yanovich
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# MapReduce
+
+MapReduce is a simple distributed MapReduce framework on Ruby.
+
+Internally there are ZMQ Transport and Evenmachine.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mapreduce'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mapreduce
+
+## Usage
+
+TODO
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+task :default => :spec
+
+Rake::TestTask.new(:spec) do |t|
+  t.libs << 'spec'
+  t.pattern = 'spec/**/*_spec.rb'
+  t.verbose = false
+end

data/lib/map_reduce.rb

@@ -0,0 +1,21 @@
+require "map_reduce/version"
+require "digest/sha1"
+require "em-zmq-tp10"
+require "logger"
+
+module MapReduce
+  DEFAULT_SOCKET = "ipc:///dev/shm/master.sock"
+
+  extend self
+
+  def logger
+    @logger ||= begin
+      log = Logger.new(STDOUT)
+      log.formatter = Logger::Formatter.new
+      log
+    end
+  end
+end
+
+require File.expand_path("../map_reduce/master", __FILE__)
+require File.expand_path("../map_reduce/worker", __FILE__)

data/lib/map_reduce/master.rb

@@ -0,0 +1,180 @@
+require File.expand_path("../socket/master", __FILE__)
+
+module MapReduce
+  class Master
+    # How often data will be flushed to disk
+    FLUSH_TIMEOUT = 1
+    # How many lines should be parsed by one iteration of grouping
+    GROUP_LINES = 100
+    # How many seconds should we sleep if grouping is going faster then reducing
+    GROUP_TIMEOUT = 1
+    # How many keys should be stored before timeout happend
+    GROUP_MAX = 10_000
+
+    # Valid options:
+    #   * socket - socket address to bind
+    #       default is 'ipc:///dev/shm/master.sock'
+    #   * log_folder - folder to store recieved MAP data
+    #       default is '/tmp/mapreduce/'
+    #   * workers - count of workers that will emit data.
+    #       default is :auto, 
+    #       but in small jobs it is better to define in explicitly,
+    #       because if one worker will stop before others start
+    #       master will decide that map job is done and will start reducing
+    #   * delimiter - master log stores data like "key{delimiter}values"
+    #       so to prevent collisions you can specify your own uniq delimiter
+    #       default is a pipe "|"
+    #
+    def initialize(opts = {})
+      # Socket addr to bind
+      @socket_addr = opts[:socket] || ::MapReduce::DEFAULT_SOCKET
+      # Folder to write logs
+      @log_folder = opts[:log_folder] || "/tmp/mapreduce/"
+      # How many MapReduce workers will emit data
+      @workers = opts[:workers] || 1
+      # Delimiter to store key/value pairs in log
+      @delimiter = opts[:delimiter] || "|"
+
+      @log = []
+      @data = []
+      @workers_envelopes = {}
+      @log_filename = File.join(@log_folder, "master-#{Process.pid}.log")
+      @sorted_log_filename = File.join(@log_folder, "master-#{Process.pid}_sorted.log")
+
+      FileUtils.mkdir_p(@log_folder)
+      FileUtils.touch(@log_filename)
+    end
+
+    # Start Eventloop
+    #
+    def run
+      EM.run do
+        # Init socket
+        master_socket
+
+        # Init flushing timer
+        flush
+      end
+    end
+
+    # Stop Eventloop
+    #
+    def stop
+      EM.stop
+    end
+
+    # Store data in log array till flush
+    #
+    def map(key, message)
+      @log << "#{key}#{@delimiter}#{message}"
+    end
+
+    # Send data back to worker.
+    # Last item in data is last unfinished session,
+    #   so till the end of file reading we don't send it
+    #
+    def reduce(envelope)
+      if @data.size >= 2 
+        data = @data.shift
+        data = data.flatten
+        master_socket.send_reply(data, envelope)
+      elsif @reduce_stop
+        data = @data.shift
+        data = data.flatten  if data
+        master_socket.send_reply(data, envelope)
+      else
+        EM.add_timer(1) do
+          reduce(envelope)
+        end
+      end
+    end
+
+    # Openning log file for read/write
+    #
+    def log_file
+      @log_file ||= begin
+        File.open(@log_filename, "w+")
+      end
+    end
+
+    # Openning sorted log for reading
+    #
+    def sorted_log_file
+      @sorted_log_file ||= begin
+        File.open(@sorted_log_filename, "r")
+      end
+    end
+
+    # Flushing data to disk once per FLUSH_TIMEOUT seconds
+    #
+    def flush
+      if @log.any?
+        log_file << @log*"\n"
+        log_file.flush
+        @log.clear
+      end
+
+      EM.add_timer(FLUSH_TIMEOUT) do
+        flush
+      end
+    end
+
+    # Sorting log.
+    # Linux sort is the fastest way to sort big file.
+    # Deleting original log after sort.
+    #
+    def sort
+      `sort #{@log_filename} -o #{@sorted_log_filename}`
+      FileUtils.rm(@log_filename)
+    end
+
+    # Start reducing part.
+    # First, flushing rest of log to disk.
+    # Then sort data.
+    # Then start to read/group data
+    #
+    def reduce!
+      flush
+      sort
+
+      iter = sorted_log_file.each_line
+      group iter
+    end
+
+    # Reading sorted data and grouping by key.
+    # If queue (@data) is growing faster then workers grad data we pause reading file.
+    #
+    def group(iter)
+      if @data.size >= GROUP_MAX
+        EM.add_timer(GROUP_TIMEOUT){ group(iter) }
+      else
+        GROUP_LINES.times do
+          line = iter.next.chomp
+          key, msg = line.split(@delimiter)
+
+          last = @data.last
+          if last && last[0] == key
+            last[1] << msg
+          else
+            @data << [key, [msg]]
+          end
+        end
+
+        EM.next_tick{ group(iter) }
+      end
+    rescue StopIteration => e
+      FileUtils.rm(@sorted_log_filename)
+      @reduce_stop = true
+    end
+
+    # Initializing and binding socket
+    #
+    def master_socket
+      @master_socket ||= begin
+        sock = MapReduce::Socket::Master.new self, @workers
+        sock.bind @socket_addr
+        sock
+      end
+    end
+  end
+end

data/lib/map_reduce/socket/master.rb

@@ -0,0 +1,100 @@
+# Reply socket.
+# Master accepts "map", "map_finished", and "reduce" messages.
+# For "map" messages it didn't actually replies,
+#   but for "reduce" requests it returns key with grouped values.
+#
+module MapReduce::Socket
+  class Master < EM::Protocols::Zmq2::Rep
+    # If worker is ready to reduce data, but we are still in MAP state
+    # we will sleep for REDUCE_WAIT seconds till state is not REDUCE
+    REDUCE_WAIT = 1
+
+    def initialize(master, workers)
+      @master = master
+      @workers = workers
+
+      @connections = {}
+      @state = :map
+
+      super()
+    end
+
+    def receive_request(message, envelope)
+      @connections[envelope.first] = false
+
+      type, key, msg = message
+      case type 
+      when "map"
+        map(envelope, key, msg)
+      when "map_finished"
+        map_finished(envelope)
+      when "reduce"
+        reduce(envelope)
+      else
+        MapReduce.logger.error("Wrong message type: #{type}")
+      end
+    end
+
+    # Send data to log
+    # Someone should never MAP data when master already in REDUCE state
+    #
+    def map(envelope, key, msg)
+      if @state == :map
+        @master.map(key, msg)
+        ok(envelope)
+      else
+        MapReduce.logger.error("Someone tries to MAP data while state is REDUCE")
+        not_ok(envelope, "You can't MAP while we are reducing")
+      end
+    end
+
+    # When worker stops mapping data, it sends "map_finished" message.
+    # When all workers will send "map_finished" message reduce will begin.
+    #
+    def map_finished(envelope)
+      ok(envelope)
+
+      @connections[envelope.first] ||= true
+      @workers = @connections.size  if @workers == :auto
+
+      return  unless @connections.all?{ |k,v| v }
+      return  unless @connections.size == @workers
+
+      @state = :reduce
+      @master.reduce!
+    end
+
+    # Wait till all workers stopps sending MAP.
+    # After all workers stopped we start REDUCE part of job.
+    #
+    def reduce(envelope)
+      @connections[envelope] ||= true
+      if @state == :reduce
+        @state == :map  unless @master.reduce(envelope)
+      else
+        EM.add_timer(REDUCE_WAIT) do
+          reduce(envelope)
+        end
+      end
+    end
+
+    # Simple OK reply
+    #
+    def ok(envelope)
+      send_reply(["ok"], envelope)
+    end
+
+    # Simple NOT OK reply
+    #
+    def not_ok(envelope, error)
+      send_reply(["error", error], envelope)
+    end
+
+    # Switch back to :map state if reduce finished
+    #
+    def send_reply(data, envelope)
+      @state = :map  unless data
+      super
+    end
+  end
+end

data/lib/map_reduce/socket/worker_em.rb

@@ -0,0 +1,4 @@
+module MapReduce::Socket
+  class WorkerEm < EM::Protocols::Zmq2::ReqCb
+  end
+end

data/lib/map_reduce/socket/worker_sync.rb

@@ -0,0 +1,16 @@
+module MapReduce::Socket
+  class WorkerSync < EM::Protocols::Zmq2::ReqCb
+    alias_method :async_send_request, :send_request
+    def send_request(data, &blk)
+      fib = Fiber.current
+      async_send_request(data) do |message|
+        fib.resume(message)
+      end
+      if block_given?
+        blk.call Fiber.yield
+      else
+        Fiber.yield
+      end
+    end
+  end
+end

data/lib/map_reduce/version.rb

@@ -0,0 +1,3 @@
+module MapReduce
+  VERSION = "0.0.1.alpha"
+end

data/lib/map_reduce/worker.rb

@@ -0,0 +1,118 @@
+# MapReduce Worker make two jobs:
+#   First, it maps (emits) all data to masters;
+#   Second, it reduces data returned form master;
+#
+module MapReduce
+  class Worker
+
+    # Valid options:
+    #   * masters - socket addresses of masters, 
+    #       default is 'ipc:///dev/shm/master.sock'
+    #   * type - connection type:
+    #     ** :em - Eventmachine with callbacks (default)
+    #     ** :sync - Synchronous type on Fibers
+    #
+    def initialize(opts = {})
+      @master_sockets = opts[:masters] || [::MapReduce::DEFAULT_SOCKET]
+
+      opts[:type] ||= :em
+      @socket_class = case opts[:type]
+      when :em
+        require File.expand_path("../socket/worker_em", __FILE__)
+        MapReduce::Socket::WorkerEm
+      when :sync
+        require File.expand_path("../socket/worker_sync", __FILE__)
+        MapReduce::Socket::WorkerSync
+      else
+        fail "Wrong Connection type. Choose :em or :sync, not #{opts[:type]}"
+      end
+    end
+
+    # Sends key and value to master through socket.
+    # Key can't be nil.
+    #
+    def emit(key, value, &blk)
+      fail "Key can't be nil"  if key.nil?
+
+      sock = pick_socket(key)
+      sock.send_request(["map", key, value], &blk)
+    end
+    alias :map :emit
+
+    # Explicitly stop MAP phase.
+    # Master will wait till all workers will send "map_finished" message.
+    #
+    def map_finished(&blk)
+      all = worker_sockets.size
+      resp = 0
+
+      worker_sockets.each do |sock|
+        sock.send_request(["map_finished"]) do |msg|
+          blk.call  if block_given? && (resp+=1) == all
+        end
+      end
+      ["ok"]
+    end
+
+    # Reduce operation.
+    # Sends request to all masters.
+    # If master returns nil it means that he is already empty:
+    #  nothing to reduce.
+    # Reducing till any socket returns data.
+    # If nothing to reduce, we return nil to client.
+    #
+    def reduce(&blk)
+      sock = random_socket
+      if sock
+        sock.send_request(["reduce"]) do |message|
+          key, *values = message
+          if key.nil?
+            remove_socket(sock)
+          else
+            blk.call(key, values)
+          end
+          reduce(&blk)
+        end
+      else
+        blk.call([nil])
+      end
+    end
+
+    private
+
+    # Connect to each master.
+    #
+    def worker_sockets
+      @worker_sockets ||= begin
+        @master_sockets.map do |addr|
+          sock = @socket_class.new
+          sock.connect addr
+          sock
+        end
+      end
+    end
+
+    # Kind of sharding
+    #
+    def pick_socket(key)
+      shard = if worker_sockets.size > 1
+        Digest::MD5.hexdigest(key.to_s).to_i(16) % worker_socket.size
+      else
+        0
+      end
+      worker_sockets[shard]
+    end
+
+    # Take random socket to get reduce message
+    #
+    def random_socket
+      worker_sockets.sample
+    end
+
+    # Remove socket when it is empty
+    #
+    def remove_socket(sock)
+      worker_sockets.delete sock
+    end
+  end
+end

data/map_reduce.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'map_reduce/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "map_reduce"
+  spec.version       = MapReduce::VERSION
+  spec.authors       = ["Petr Yanovich"]
+  spec.email         = ["fl00r@yandex.ru"]
+  spec.description   = %q{Simple distributed Map Reduce Framework on Ruby}
+  spec.summary       = %q{Simple distributed Map Reduce Framework on Ruby}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "em-synchrony"
+
+  spec.add_dependency "em-zmq-tp10"
+end

data/spec/map_reduce/map_reduce_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe "MapReduce stack" do
+  before do
+    @pid = fork do
+      master = MapReduce::Master.new
+      master.run
+    end
+  end
+
+  after do
+    Process.kill "TERM", @pid
+  end
+
+  it "should map and reduce some data in CB mode" do
+    EM.run do
+      data = {}
+      worker = MapReduce::Worker.new
+      worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',') do
+        worker.map("Alex", ["Madonna", "Lady Gaga"] * ',') do
+          worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',') do
+            worker.map_finished do
+              worker.reduce do |key, values|
+                if key
+                  data[key] = values
+                else
+                  data.size.must_equal 2
+                  data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
+                  data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
+
+                  EM.stop
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  it "should map and reduce some data in SYNC mode" do
+    EM.synchrony do
+      data = {}
+      worker = MapReduce::Worker.new type: :sync
+      worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',')
+      worker.map("Alex", ["Madonna", "Lady Gaga"] * ',')
+      worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',')
+      worker.map_finished
+      worker.reduce do |key, values|
+        data[key] = values  if key
+      end
+      data.size.must_equal 2
+      data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
+      data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
+
+      EM.stop
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+LIB_PATH = File.expand_path('../../lib/map_reduce',  __FILE__)
+
+require LIB_PATH
+
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'em-synchrony'

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: map_reduce
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha
+  prerelease: 6
+platform: ruby
+authors:
+- Petr Yanovich
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    none: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    none: false
+  prerelease: false
+  name: bundler
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  prerelease: false
+  name: rake
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  prerelease: false
+  name: em-synchrony
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  prerelease: false
+  name: em-zmq-tp10
+description: Simple distributed Map Reduce Framework on Ruby
+email:
+- fl00r@yandex.ru
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/map_reduce.rb
+- lib/map_reduce/master.rb
+- lib/map_reduce/socket/master.rb
+- lib/map_reduce/socket/worker_em.rb
+- lib/map_reduce/socket/worker_sync.rb
+- lib/map_reduce/version.rb
+- lib/map_reduce/worker.rb
+- map_reduce.gemspec
+- spec/map_reduce/map_reduce_spec.rb
+- spec/map_reduce/master_spec.rb
+- spec/map_reduce/worker_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      hash: 2247563880137465180
+      version: '0'
+      segments:
+      - 0
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+  none: false
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Simple distributed Map Reduce Framework on Ruby
+test_files:
+- spec/map_reduce/map_reduce_spec.rb
+- spec/map_reduce/master_spec.rb
+- spec/map_reduce/worker_spec.rb
+- spec/spec_helper.rb