map_reduce 0.0.4 → 0.0.5

data/lib/map_reduce.rb

@@ -24,4 +24,4 @@ require File.expand_path("../map_reduce/reduce_log", __FILE__)
 require File.expand_path("../map_reduce/socket/master", __FILE__)
 require File.expand_path("../map_reduce/master", __FILE__)
 require File.expand_path("../map_reduce/mapper", __FILE__)
-require File.expand_path("../map_reduce/reducer", __FILE__)
+require File.expand_path("../map_reduce/reducer", __FILE__)

data/lib/map_reduce/map_log.rb

@@ -40,9 +40,7 @@ module MapReduce
 
     def log_file
       @log_file ||= begin
-        begin
-          fn = File.join(@log_folder, "mapper.log")
-        end while File.exist?(fn)
+        fn = File.join(@log_folder, "mapper.log")
         FileUtils.mkdir_p(@log_folder)
         File.open(fn, "a")
       end

data/lib/map_reduce/mapper.rb

@@ -4,13 +4,26 @@ module MapReduce
       @masters         = opts[:masters] || [::MapReduce::DEFAULT_SOCKET]
       @connection_type = opts[:type]    || :em
       @task_name       = opts[:task]
+      @disconnected    = {}
     end
 
     def emit(key, value, &blk)
       raise MapReduce::Exceptions::BlankKey, "Key can't be nil"  if key.nil?
 
       sock = pick_master(key)
-      sock.send_request(["map", key, value, @task_name], &blk)
+      sock.send_request(["map", key, value, @task_name]) do |res|
+        if res
+          @disconnected.delete(sock)  if @disconnected[sock]
+          if blk
+            blk.call(res)
+          else 
+            return res
+          end
+        else
+          @disconnected[sock] = true
+          emit(key, value, &blk)
+        end
+      end
     end
     alias :map :emit
 
@@ -50,7 +63,13 @@ module MapReduce
 
     def pick_master(key)
       num = Digest::MD5.hexdigest(key.to_s).to_i(16) % sockets.size
-      sockets[num]
+      sock = sockets[num]
+      # LOL :)
+      if @disconnected[sock] && rand(10) != 0
+        pick_master(key.chars.to_a.shuffle.join)
+      else
+        sock
+      end
     end
 
     def sockets

data/lib/map_reduce/reduce_log.rb

@@ -56,7 +56,7 @@ module MapReduce
     end
 
     def sort(fn)
-      `sort #{fn} -o #{fn}`
+      `sort #{fn} -o #{fn} -k1nr`
     end
   end
 end

data/lib/map_reduce/reducer.rb

@@ -1,5 +1,7 @@
 module MapReduce
   class Reducer
+    TIMEOUT = 0.1
+
     def initialize(opts = {})
       @masters         = opts[:masters] || [::MapReduce::DEFAULT_SOCKET]
       @connection_type = opts[:type]    || :em

data/lib/map_reduce/socket/req_fiber.rb

@@ -2,9 +2,17 @@ module EM::Protocols::Zmq2
   class ReqFiber < EM::Protocols::Zmq2::ReqCb
     def send_request(data, &blk)
       fib = Fiber.current
-      super(data) do |message|
+      timer = nil
+      request_id = super(data) do |message|
+        EM.cancel_timer(timer)
         fib.resume(message)
       end
+      if request_id
+        timer = EM.add_timer(1) {
+          MapReduce.logger.info("TIMEOUT")
+          cancel_request(request_id)
+        }
+      end
       if block_given?
         blk.call Fiber.yield
       else

data/lib/map_reduce/version.rb

@@ -1,3 +1,3 @@
 module MapReduce
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/spec/map_reduce/map_reduce_spec.rb

@@ -137,6 +137,28 @@ describe "MapReduce stack" do
           EM.stop
         end
       end
+
+      it "should use one master" do
+        EM.synchrony do
+          # MapReduce::Reducer.const_set :TIMEOUT, 0.1
+          Process.kill "INT", @pid2
+          @mapper = MapReduce::Mapper.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          @reducer = MapReduce::Reducer.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          [["Peter", "Apple"], ["Andrew", "Peach"], ["Mary", "Plum"], ["Peter", "Lemon"], ["Andrew", "Orange"]].each do |a|
+            res = @mapper.map(*a)
+            res.must_equal ["ok"]
+          end
+          data = {}
+          @reducer.reduce do |k, values|
+            data[k] = values
+          end
+          data.size.must_equal 3
+          data["Peter"].sort.must_equal ["Apple", "Lemon"].sort
+          data["Andrew"].sort.must_equal ["Peach", "Orange"].sort
+          data["Mary"].must_equal ["Plum"]
+          EM.stop
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: map_reduce
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-06 00:00:00.000000000 Z
+date: 2013-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -117,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2639793831628664557
+      hash: -2358155464795184173
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -126,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2639793831628664557
+      hash: -2358155464795184173
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25