map_reduce 0.0.1.alpha2 → 0.0.1.alpha3

data/lib/map_reduce/master.rb

@@ -126,6 +126,7 @@ module MapReduce
     def sort
       `sort #{@log_filename} -o #{@sorted_log_filename}`
       FileUtils.rm(@log_filename)
+      @log_file = nil
     end
 
     # Start reducing part.
@@ -164,6 +165,7 @@ module MapReduce
       end
     rescue StopIteration => e
       FileUtils.rm(@sorted_log_filename)
+      @sorted_log_file = nil
       @reduce_stop = true
     end
 

data/lib/map_reduce/socket/master.rb

@@ -64,13 +64,12 @@ module MapReduce::Socket
       @master.reduce!
     end
 
-    # Wait till all workers stopps sending MAP.
+    # Wait till all workers stops sending MAP.
     # After all workers stopped we start REDUCE part of job.
     #
     def reduce(envelope)
-      @connections[envelope] ||= true
       if @state == :reduce
-        @state == :map  unless @master.reduce(envelope)
+        @master.reduce(envelope)
       else
         EM.add_timer(REDUCE_WAIT) do
           reduce(envelope)

data/lib/map_reduce/version.rb

@@ -1,3 +1,3 @@
 module MapReduce
-  VERSION = "0.0.1.alpha2"
+  VERSION = "0.0.1.alpha3"
 end

data/lib/map_reduce/worker.rb

@@ -1,6 +1,7 @@
 # MapReduce Worker make two jobs:
 #   First, it maps (emits) all data to masters;
 #   Second, it reduces data returned form master;
+# After reducing he is ready to map data again.
 #
 module MapReduce
   class Worker
@@ -13,10 +14,10 @@ module MapReduce
     #     ** :sync - Synchronous type on Fibers
     #
     def initialize(opts = {})
-      @master_sockets = opts[:masters] || [::MapReduce::DEFAULT_SOCKET]
+      @socket_addrs = opts[:masters] || [::MapReduce::DEFAULT_SOCKET]
 
-      opts[:type] ||= :em
-      @socket_class = case opts[:type]
+      @type = opts[:type] ||= :em
+      @socket_class = case @type
       when :em
         require File.expand_path("../socket/worker_em", __FILE__)
         MapReduce::Socket::WorkerEm
@@ -34,7 +35,7 @@ module MapReduce
     def emit(key, value, &blk)
       fail "Key can't be nil"  if key.nil?
 
-      sock = pick_socket(key)
+      sock = pick_map_socket(key)
       sock.send_request(["map", key, value], &blk)
     end
     alias :map :emit
@@ -43,12 +44,13 @@ module MapReduce
     # Master will wait till all workers will send "map_finished" message.
     #
     def map_finished(&blk)
-      all = worker_sockets.size
+      all = master_sockets.size
       resp = 0
 
-      worker_sockets.each do |sock|
+      master_sockets.each do |sock, h|
         sock.send_request(["map_finished"]) do |msg|
-          blk.call  if block_given? && (resp+=1) == all
+          socket_state(sock, :reduce)
+          blk.call(["ok"])  if block_given? && (resp+=1) == all
         end
       end
       ["ok"]
@@ -62,16 +64,36 @@ module MapReduce
     # If nothing to reduce, we return nil to client.
     #
     def reduce(&blk)
-      sock = random_socket
+      if @type == :em
+        em_reduce(&blk)
+      else
+        sync_reduce(&blk)
+      end
+    end
+
+    def sync_reduce(&blk)
+      while sock = random_reduce_socket
+        key, *values = sock.send_request(["reduce"])
+        if key.nil?
+          socket_state(sock, :map)
+        else
+          blk.call(key, values)
+        end
+      end
+    end
+
+    def em_reduce(&blk)
+      sock = random_reduce_socket
       if sock
         sock.send_request(["reduce"]) do |message|
           key, *values = message
           if key.nil?
-            remove_socket(sock)
+            socket_state(sock, :map)
           else
             blk.call(key, values)
           end
-          reduce(&blk)
+
+          em_reduce(&blk)
         end
       else
         blk.call([nil])
@@ -82,37 +104,41 @@ module MapReduce
 
     # Connect to each master.
     #
-    def worker_sockets
-      @worker_sockets ||= begin
-        @master_sockets.map do |addr|
+    def master_sockets
+      @master_sockets ||= begin
+        socks = {}
+        @socket_addrs.each_with_index do |addr, i|
           sock = @socket_class.new
           sock.connect addr
-          sock
+          socks[sock] = { state: :map, ind: i }
         end
+        socks
       end
     end
 
     # Kind of sharding
     #
-    def pick_socket(key)
-      shard = if worker_sockets.size > 1
-        Digest::MD5.hexdigest(key.to_s).to_i(16) % worker_sockets.size
+    def pick_map_socket(key)
+      shard = if master_sockets.size > 1
+        Digest::MD5.hexdigest(key.to_s).to_i(16) % master_sockets.size
       else
         0
       end
-      worker_sockets[shard]
+      master_sockets.keys[shard]
     end
 
-    # Take random socket to get reduce message
+    # Take random socket to get reduce message.
+    # Socket should be in :reduce state.
     #
-    def random_socket
-      worker_sockets.sample
+    def random_reduce_socket
+      master_sockets.select{ |k,v| v[:state] == :reduce }.keys.sample
     end
 
-    # Remove socket when it is empty
+    # Change socket's state to :map when it is empty
+    #   and to :reduce when mapping is finished
     #
-    def remove_socket(sock)
-      worker_sockets.delete sock
+    def socket_state(sock, state)
+      master_sockets[sock][:state] = state
     end
   end
 end

data/spec/map_reduce/map_reduce_spec.rb

@@ -76,23 +76,25 @@ describe "MapReduce stack" do
       Process.kill "TERM", @pid2
     end
 
-    it "should map and reduce some data in SYNC mode" do
+    it "should map and reduce some data in SYNC mode twice" do
       EM.synchrony do
-        data = {}
         worker = MapReduce::Worker.new type: :sync,  masters: ["ipc:///dev/shm/sock1.sock", "ipc:///dev/shm/sock2.sock"]
-        worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',')
-        worker.map("Alex", ["Madonna", "Lady Gaga"] * ',')
-        worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',')
-        worker.map("Michael", ["Blur"] * ',')
-        worker.map("Gosha", ["DDT", "Splin"] * ',')
-        worker.map("Obama", ["Adele", "Rolling Stones"] * ',')
-        worker.map_finished
-        worker.reduce do |key, values|
-          data[key] = values  if key
+        2.times do
+          data = {}
+          worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',')
+          worker.map("Alex", ["Madonna", "Lady Gaga"] * ',')
+          worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',')
+          worker.map("Michael", ["Blur"] * ',')
+          worker.map("Gosha", ["DDT", "Splin"] * ',')
+          worker.map("Obama", ["Adele", "Rolling Stones"] * ',')
+          worker.map_finished
+          worker.reduce do |key, values|
+            data[key] = values  if key
+          end
+          data.size.must_equal 5
+          data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
+          data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
         end
-        data.size.must_equal 5
-        data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
-        data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
 
         EM.stop
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: map_reduce
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha2
+  version: 0.0.1.alpha3
   prerelease: 6
 platform: ruby
 authors:
@@ -110,7 +110,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      hash: 2663364217523840830
+      hash: 1980425592247416328
       version: '0'
       segments:
       - 0