map_reduce 0.0.1.alpha4 → 0.0.1.alpha5

data/lib/map_reduce/master_old.rb

@@ -0,0 +1,182 @@
+require File.expand_path("../socket/master", __FILE__)
+
+module MapReduce
+  class Master
+    # How often data will be flushed to disk
+    FLUSH_TIMEOUT = 1
+    # How many lines should be parsed by one iteration of grouping
+    GROUP_LINES = 100
+    # How many seconds should we sleep if grouping is going faster then reducing
+    GROUP_TIMEOUT = 1
+    # How many keys should be stored before timeout happend
+    GROUP_MAX = 10_000
+
+    # Valid options:
+    #   * socket - socket address to bind
+    #       default is 'ipc:///dev/shm/master.sock'
+    #   * log_folder - folder to store recieved MAP data
+    #       default is '/tmp/mapreduce/'
+    #   * workers - count of workers that will emit data.
+    #       default is :auto, 
+    #       but in small jobs it is better to define in explicitly,
+    #       because if one worker will stop before others start
+    #       master will decide that map job is done and will start reducing
+    #   * delimiter - master log stores data like "key{delimiter}values"
+    #       so to prevent collisions you can specify your own uniq delimiter
+    #       default is a pipe "|"
+    #
+    def initialize(opts = {})
+      # Socket addr to bind
+      @socket_addr = opts[:socket] || ::MapReduce::DEFAULT_SOCKET
+      # Folder to write logs
+      @log_folder = opts[:log_folder] || "/tmp/mapreduce/"
+      # How many MapReduce workers will emit data
+      @workers = opts[:workers] || 1
+      # Delimiter to store key/value pairs in log
+      @delimiter = opts[:delimiter] || "|"
+
+      @log = []
+      @data = []
+      @workers_envelopes = {}
+      @log_filename = File.join(@log_folder, "master-#{Process.pid}.log")
+      @sorted_log_filename = File.join(@log_folder, "master-#{Process.pid}_sorted.log")
+
+      FileUtils.mkdir_p(@log_folder)
+      FileUtils.touch(@log_filename)
+    end
+
+    # Start Eventloop
+    #
+    def run
+      EM.run do
+        # Init socket
+        master_socket
+
+        # Init flushing timer
+        flush
+      end
+    end
+
+    # Stop Eventloop
+    #
+    def stop
+      EM.stop
+    end
+
+    # Store data in log array till flush
+    #
+    def map(key, message)
+      @log << "#{key}#{@delimiter}#{message}"
+    end
+
+    # Send data back to worker.
+    # Last item in data is last unfinished session,
+    #   so till the end of file reading we don't send it
+    #
+    def reduce(envelope)
+      if @data.size >= 2 
+        data = @data.shift
+        data = data.flatten
+        master_socket.send_reply(data, envelope)
+      elsif @reduce_stop
+        data = @data.shift
+        data = data.flatten  if data
+        master_socket.send_reply(data, envelope)
+      else
+        EM.add_timer(1) do
+          reduce(envelope)
+        end
+      end
+    end
+
+    # Openning log file for read/write
+    #
+    def log_file
+      @log_file ||= begin
+        File.open(@log_filename, "w+")
+      end
+    end
+
+    # Openning sorted log for reading
+    #
+    def sorted_log_file
+      @sorted_log_file ||= begin
+        File.open(@sorted_log_filename, "r")
+      end
+    end
+
+    # Flushing data to disk once per FLUSH_TIMEOUT seconds
+    #
+    def flush
+      if @log.any?
+        log_file << @log*"\n" << "\n"
+        log_file.flush
+        @log.clear
+      end
+
+      EM.add_timer(FLUSH_TIMEOUT) do
+        flush
+      end
+    end
+
+    # Sorting log.
+    # Linux sort is the fastest way to sort big file.
+    # Deleting original log after sort.
+    #
+    def sort
+      `sort #{@log_filename} -o #{@sorted_log_filename}`
+      FileUtils.rm(@log_filename)
+      @log_file = nil
+    end
+
+    # Start reducing part.
+    # First, flushing rest of log to disk.
+    # Then sort data.
+    # Then start to read/group data
+    #
+    def reduce!
+      flush
+      sort
+
+      iter = sorted_log_file.each_line
+      group iter
+    end
+
+    # Reading sorted data and grouping by key.
+    # If queue (@data) is growing faster then workers grad data we pause reading file.
+    #
+    def group(iter)
+      if @data.size >= GROUP_MAX
+        EM.add_timer(GROUP_TIMEOUT){ group(iter) }
+      else
+        GROUP_LINES.times do
+          line = iter.next.chomp
+          key, msg = line.split(@delimiter)
+
+          last = @data.last
+          if last && last[0] == key
+            last[1] << msg
+          else
+            @data << [key, [msg]]
+          end
+        end
+
+        EM.next_tick{ group(iter) }
+      end
+    rescue StopIteration => e
+      FileUtils.rm(@sorted_log_filename)
+      @sorted_log_file = nil
+      @reduce_stop = true
+    end
+
+    # Initializing and binding socket
+    #
+    def master_socket
+      @master_socket ||= begin
+        sock = MapReduce::Socket::Master.new self, @workers
+        sock.bind @socket_addr
+        sock
+      end
+    end
+  end
+end

data/lib/map_reduce/reduce_log.rb

@@ -0,0 +1,60 @@
+module MapReduce
+  class ReduceLog
+    def initialize(map_log, delimiter)
+      @map_log = map_log
+      @delimiter = delimiter
+    end
+
+    def get_data
+      if @lines
+        current_key = nil
+        current_values = []
+        while true
+          begin
+            line = @lines.peek.chomp
+            key, values = line.split(@delimiter)
+            current_key ||= key
+
+            if current_key != key
+              break
+            else
+              current_values << values
+              @lines.next
+            end
+          rescue StopIteration => e
+            @file.close
+            FileUtils.rm(File.path(@file))
+            @lines = nil
+            break
+          end
+        end
+        [current_key, *current_values]
+      end
+    end
+
+    def force
+      unless @lines
+        fn = log_file
+        if fn
+          @file = File.open(fn)
+          @lines = @file.each_line
+        end
+      end
+    end
+
+    def log_file
+      @log_file ||= begin
+        fn = @map_log.reset
+        if fn
+          @more = true
+          sort(fn)
+          fn
+        end
+      end
+    end
+
+    def sort(fn)
+      `sort #{fn} -o #{fn}`
+    end
+  end
+end

data/lib/map_reduce/reducer.rb

@@ -0,0 +1,65 @@
+module MapReduce
+  class Reducer
+    def initialize(opts = {})
+      @masters         = opts[:masters] || [::MapReduce::DEFAULT_SOCKET]
+      @connection_type = opts[:type]    || :em
+      @task            = opts[:task]
+    end
+
+    def reduce(&blk)
+      if @connection_type == :em
+        em_reduce(&blk)
+      else
+        sync_reduce(&blk)
+      end
+    end
+
+    def sync_reduce(&blk)
+      all = sockets.dup
+      while sock = all.sample
+        key, *values = sock.send_request(["reduce", @task])
+        if key.nil?
+          all.delete sock
+        else
+          blk.call(key, values)
+        end
+      end
+    end
+
+    def em_reduce(all = nil, &blk)
+      all ||= sockets.dup
+      sock = all.sample
+      if sock
+        sock.send_request(["reduce", @task]) do |message|
+          key, *values = message
+          if key.nil?
+            all.delete sock
+          else
+            blk.call(key, values)
+          end
+
+          em_reduce(all, &blk)
+        end
+      else
+        blk.call([nil])
+      end
+    end
+    private
+
+    def sockets
+      @sockets ||= begin
+        klass = if @connection_type == :sync
+          EM::Protocols::Zmq2::ReqFiber
+        else
+          EM::Protocols::Zmq2::ReqCb
+        end
+
+        @masters.map do |sock|
+          s = klass.new
+          s.connect(sock)
+          s
+        end
+      end
+    end
+  end
+end

data/lib/map_reduce/socket/master.rb

@@ -1,102 +1,12 @@
-# Reply socket.
-# Master accepts "map", "map_finished", and "reduce" messages.
-# For "map" messages it didn't actually replies,
-#   but for "reduce" requests it returns key with grouped values.
-#
 module MapReduce::Socket
   class Master < EM::Protocols::Zmq2::Rep
-    # If worker is ready to reduce data, but we are still in MAP state
-    # we will sleep for REDUCE_WAIT seconds till state is not REDUCE
-    REDUCE_WAIT = 1
-
-    def initialize(master, workers)
+    def initialize(master)
       @master = master
-      @workers = workers
-
-      @connections = {}
-      @state = :map
-
       super()
     end
 
     def receive_request(message, envelope)
-      @connections[envelope.first] = false
-
-      type, key, msg = message
-      case type 
-      when "map"
-        map(envelope, key, msg)
-      when "map_finished"
-        map_finished(envelope)
-      when "reduce"
-        reduce(envelope)
-      else
-        MapReduce.logger.error("Wrong message type: #{type}")
-      end
-    end
-
-    # Send data to log
-    # Someone should never MAP data when master already in REDUCE state
-    #
-    def map(envelope, key, msg)
-      if @state == :map
-        @master.map(key, msg)
-        ok(envelope)
-      else
-        MapReduce.logger.error("Someone tries to MAP data while state is REDUCE")
-        not_ok(envelope, "You can't MAP while we are reducing")
-      end
-    end
-
-    # When worker stops mapping data, it sends "map_finished" message.
-    # When all workers will send "map_finished" message reduce will begin.
-    #
-    def map_finished(envelope)
-      ok(envelope)
-
-      @connections[envelope.first] ||= true
-      @workers = @connections.size  if @workers == :auto
-
-      return  unless @connections.all?{ |k,v| v }
-      return  unless @connections.size == @workers
-
-      @state = :reduce
-      @master.reduce!
-    end
-
-    # Wait till all workers stops sending MAP.
-    # After all workers stopped we start REDUCE part of job.
-    #
-    def reduce(envelope)
-      if @state == :reduce
-        @master.reduce(envelope)
-      else
-        EM.add_timer(REDUCE_WAIT) do
-          reduce(envelope)
-        end
-      end
-    end
-
-    # Simple OK reply
-    #
-    def ok(envelope)
-      send_reply(["ok"], envelope)
-    end
-
-    # Simple NOT OK reply
-    #
-    def not_ok(envelope, error)
-      send_reply(["error", error], envelope)
-    end
-
-    # Switch back to :map state if reduce finished
-    #
-    def send_reply(data, envelope)
-      unless data
-        @state = :map
-        @connections = {}
-      end
-      super
+      @master.recieve_msg(message, envelope)
     end
   end
 end

data/lib/map_reduce/socket/{worker_sync.rb → req_fiber.rb}

@@ -1,9 +1,8 @@
-module MapReduce::Socket
-  class WorkerSync < EM::Protocols::Zmq2::ReqCb
-    alias_method :async_send_request, :send_request
+module EM::Protocols::Zmq2
+  class ReqFiber < EM::Protocols::Zmq2::ReqCb
     def send_request(data, &blk)
       fib = Fiber.current
-      async_send_request(data) do |message|
+      super(data) do |message|
         fib.resume(message)
       end
       if block_given?

data/lib/map_reduce/version.rb

@@ -1,3 +1,3 @@
 module MapReduce
-  VERSION = "0.0.1.alpha4"
+  VERSION = "0.0.1.alpha5"
 end

data/spec/map_reduce/map_reduce_spec.rb

@@ -3,32 +3,48 @@ require 'spec_helper'
 describe "MapReduce stack" do
   describe "single master" do
     before do
-      @pid = fork do
-        master = MapReduce::Master.new
+      @pid1 = fork do
+        master = MapReduce::Master.new socket: "tcp://127.0.0.1:15555"
+        trap("SIGINT") do
+          master.stop
+          exit
+        end
+        master.run
+      end
+      @pid2 = fork do
+        master = MapReduce::Master.new socket: "tcp://127.0.0.1:15556"
+        trap("SIGINT") do
+          master.stop
+          exit
+        end
         master.run
       end
     end
 
     after do
-      Process.kill "TERM", @pid
+      Process.kill "INT", @pid1
+      Process.kill "INT", @pid2
     end
 
-    it "should map and reduce some data in CB mode" do
-      EM.run do
-        data = {}
-        worker = MapReduce::Worker.new
-        worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',') do
-          worker.map("Alex", ["Madonna", "Lady Gaga"] * ',') do
-            worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',') do
-              worker.map_finished do
-                worker.reduce do |key, values|
+    describe ":em" do
+      it "should map/reduce with multiple masters" do
+        EM.run do
+          @mapper = MapReduce::Mapper.new task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          @reducer = MapReduce::Reducer.new task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          i = 0
+          [["Peter", "Apple"], ["Andrew", "Peach"], ["Mary", "Plum"], ["Peter", "Lemon"], ["Andrew", "Orange"]].each do |a|
+            @mapper.map(*a) do |res|
+              res.must_equal ["ok"]
+              if (i+=1) == 5
+                data = {}
+                @reducer.reduce do |key, values|
                   if key
                     data[key] = values
                   else
-                    data.size.must_equal 2
-                    data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
-                    data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
-
+                    data.size.must_equal 3
+                    data["Peter"].sort.must_equal ["Apple", "Lemon"].sort
+                    data["Andrew"].sort.must_equal ["Peach", "Orange"].sort
+                    data["Mary"].must_equal ["Plum"]
                     EM.stop
                   end
                 end
@@ -39,64 +55,61 @@ describe "MapReduce stack" do
       end
     end
 
-    it "should map and reduce some data in SYNC mode" do
-      EM.synchrony do
-        data = {}
-        worker = MapReduce::Worker.new type: :sync
-        worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',')
-        worker.map("Alex", ["Madonna", "Lady Gaga"] * ',')
-        worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',')
-        worker.map_finished
-        worker.reduce do |key, values|
-          data[key] = values  if key
+    describe ":sync" do
+      it "should map/reduce with multiple masters" do
+        EM.synchrony do
+          @mapper = MapReduce::Mapper.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          @reducer = MapReduce::Reducer.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          [["Peter", "Apple"], ["Andrew", "Peach"], ["Mary", "Plum"], ["Peter", "Lemon"], ["Andrew", "Orange"]].each do |a|
+            res = @mapper.map(*a)
+            res.must_equal ["ok"]
+          end
+          data = {}
+          @reducer.reduce do |k, values|
+            data[k] = values
+          end
+          data.size.must_equal 3
+          data["Peter"].sort.must_equal ["Apple", "Lemon"].sort
+          data["Andrew"].sort.must_equal ["Peach", "Orange"].sort
+          data["Mary"].must_equal ["Plum"]
+          EM.stop
         end
-        data.size.must_equal 2
-        data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
-        data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
-
-        EM.stop
       end
-    end
-  end
 
-  describe "multiple master" do
-    before do
-      @pid1 = fork do
-        master = MapReduce::Master.new socket: "ipc:///dev/shm/sock1.sock"
-        master.run
-      end
-      @pid2 = fork do
-        master = MapReduce::Master.new socket: "ipc:///dev/shm/sock2.sock"
-        master.run
-      end
-    end
+      it "should map/reduce-map/reduce with multiple masters" do
+        EM.synchrony do
+          @mapper1 = MapReduce::Mapper.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          @reducer1 = MapReduce::Reducer.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          @mapper2 = MapReduce::Mapper.new type: :sync, task: "Related", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          @reducer2 = MapReduce::Reducer.new type: :sync, task: "Related", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
 
-    after do
-      Process.kill "TERM", @pid1
-      Process.kill "TERM", @pid2
-    end
+          [["Peter", "Apple"], ["Andrew", "Peach"], ["Mary", "Plum"], ["Peter", "Lemon"], ["Andrew", "Orange"], ["Peter", "Peach"], ["Yura", "Peach"], ["Yura", "Apricot"], ["Yura", "Apple"]].each do |a|
+            res = @mapper1.map(*a)
+            res.must_equal ["ok"]
+          end
 
-    it "should map and reduce some data in SYNC mode twice" do
-      EM.synchrony do
-        worker = MapReduce::Worker.new type: :sync,  masters: ["ipc:///dev/shm/sock1.sock", "ipc:///dev/shm/sock2.sock"]
-        2.times do
-          data = {}
-          worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',')
-          worker.map("Alex", ["Madonna", "Lady Gaga"] * ',')
-          worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',')
-          worker.map("Michael", ["Blur"] * ',')
-          worker.map("Gosha", ["DDT", "Splin"] * ',')
-          worker.map("Obama", ["Adele", "Rolling Stones"] * ',')
-          worker.map_finished
-          worker.reduce do |key, values|
-            data[key] = values  if key
+          @reducer1.reduce do |k, values|
+            values.each do |fruit|
+              related = values.dup
+              related.delete fruit
+              related.each do |r|
+                @mapper2.map(fruit, r)
+              end
+            end
           end
-          data.size.must_equal 5
-          data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
-          data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
-        end
 
-        EM.stop
+          fruits = {}
+          @reducer2.reduce do |fruit, related|
+            fruits[fruit] ||= []
+            fruits[fruit].push(*related)
+          end
+
+          fruits["Apple"].must_equal ["Apricot", "Lemon", "Peach", "Peach"]
+          fruits["Orange"].must_equal ["Peach"]
+          fruits["Plum"].must_equal nil
+
+          EM.stop
+        end
       end
     end
   end