map_reduce 0.0.1.alpha5 → 0.0.2

data/lib/map_reduce/map_log.rb

@@ -1,6 +1,6 @@
 module MapReduce
   class MapLog
-    MAX_BUFFER_SIZE = 2 ** 20
+    MAX_BUFFER_SIZE = 2 ** 21 # 2 MB
 
     def initialize(log_folder, task)
       @log_folder = log_folder
@@ -19,6 +19,7 @@ module MapReduce
       unless @log.empty?
         log_file << @log
         log_file.flush
+        @log.clear
       end
     end
 
@@ -35,7 +36,7 @@ module MapReduce
     def log_file
       @log_file ||= begin
         begin
-          fn = File.join(@log_folder, "map_#{@task}_#{Time.now.to_i}_#{rand(1000)}.log")
+          fn = File.join(@log_folder, "map_#{@task}_#{Time.now.to_i}_#{Process.pid}_#{rand(1000)}.log")
         end while File.exist?(fn)
         FileUtils.mkdir_p(@log_folder)
         File.open(fn, "a")

data/lib/map_reduce/master.rb

@@ -61,14 +61,14 @@ module MapReduce
         reduce_log(task, true).get_data
       end
 
-      reply(data, envelope)
-
       if data
         register(task, envelope, "reducer", status)
       else
         register(task, envelope, "reducer", "reduce_finished")
       end
 
+      reply(data, envelope)
+
       @after_reduce.call(data[0], data[1], task)  if data && @after_reduce
     end
 
@@ -89,9 +89,9 @@ module MapReduce
 
     def reduce_log(task, force = false)
       @reduce_log ||= {}
-      log = @reduce_log[task] ||= MapReduce::ReduceLog.new(map_log(task), @delimiter)
+      @reduce_log[task] ||= MapReduce::ReduceLog.new(map_log(task), @delimiter)
       @reduce_log[task].force  if force
-      log
+      @reduce_log[task]
     end
 
     def ok(envelope)

data/lib/map_reduce/reduce_log.rb

@@ -43,13 +43,11 @@ module MapReduce
     end
 
     def log_file
-      @log_file ||= begin
-        fn = @map_log.reset
-        if fn
-          @more = true
-          sort(fn)
-          fn
-        end
+      fn = @map_log.reset
+      if fn
+        @more = true
+        sort(fn)
+        fn
       end
     end
 

data/lib/map_reduce/version.rb

@@ -1,3 +1,3 @@
 module MapReduce
-  VERSION = "0.0.1.alpha5"
+  VERSION = "0.0.2"
 end

data/spec/map_reduce/map_reduce_spec.rb

@@ -76,6 +76,30 @@ describe "MapReduce stack" do
         end
       end
 
+      it "should map -> reduce / reduce" do
+        EM.synchrony do
+          @mapper = MapReduce::Mapper.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+          @reducer = MapReduce::Reducer.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]
+
+          Fiber.new do
+            100.times do |i|
+              @mapper.map(i, 1)
+            end
+          end.resume
+          data = []
+          Fiber.new do
+            while data.size < 100
+              @reducer.reduce do |k, v|
+                data << k
+              end
+            end
+            data.sort.must_equal (0...100).to_a.map(&:to_s).sort
+            
+            EM.stop
+          end.resume
+        end
+      end
+
       it "should map/reduce-map/reduce with multiple masters" do
         EM.synchrony do
           @mapper1 = MapReduce::Mapper.new type: :sync, task: "Fruits", masters: ["tcp://127.0.0.1:15555", "tcp://127.0.0.1:15556"]

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: map_reduce
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha5
-  prerelease: 6
+  version: 0.0.2
+  prerelease: 
 platform: ruby
 authors:
 - Petr Yanovich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-02 00:00:00.000000000 Z
+date: 2013-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -92,13 +92,11 @@ files:
 - lib/map_reduce/map_log.rb
 - lib/map_reduce/mapper.rb
 - lib/map_reduce/master.rb
-- lib/map_reduce/master_old.rb
 - lib/map_reduce/reduce_log.rb
 - lib/map_reduce/reducer.rb
 - lib/map_reduce/socket/master.rb
 - lib/map_reduce/socket/req_fiber.rb
 - lib/map_reduce/version.rb
-- lib/map_reduce/worker.rb
 - map_reduce.gemspec
 - spec/map_reduce/map_reduce_spec.rb
 - spec/map_reduce/master_spec.rb
@@ -119,13 +117,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3538709234591680598
+      hash: 1253589631458738880
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
+      segments:
+      - 0
+      hash: 1253589631458738880
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25

data/lib/map_reduce/master_old.rb

@@ -1,182 +0,0 @@
-require File.expand_path("../socket/master", __FILE__)
-
-module MapReduce
-  class Master
-    # How often data will be flushed to disk
-    FLUSH_TIMEOUT = 1
-    # How many lines should be parsed by one iteration of grouping
-    GROUP_LINES = 100
-    # How many seconds should we sleep if grouping is going faster then reducing
-    GROUP_TIMEOUT = 1
-    # How many keys should be stored before timeout happend
-    GROUP_MAX = 10_000
-
-    # Valid options:
-    #   * socket - socket address to bind
-    #       default is 'ipc:///dev/shm/master.sock'
-    #   * log_folder - folder to store recieved MAP data
-    #       default is '/tmp/mapreduce/'
-    #   * workers - count of workers that will emit data.
-    #       default is :auto, 
-    #       but in small jobs it is better to define in explicitly,
-    #       because if one worker will stop before others start
-    #       master will decide that map job is done and will start reducing
-    #   * delimiter - master log stores data like "key{delimiter}values"
-    #       so to prevent collisions you can specify your own uniq delimiter
-    #       default is a pipe "|"
-    #
-    def initialize(opts = {})
-      # Socket addr to bind
-      @socket_addr = opts[:socket] || ::MapReduce::DEFAULT_SOCKET
-      # Folder to write logs
-      @log_folder = opts[:log_folder] || "/tmp/mapreduce/"
-      # How many MapReduce workers will emit data
-      @workers = opts[:workers] || 1
-      # Delimiter to store key/value pairs in log
-      @delimiter = opts[:delimiter] || "|"
-
-      @log = []
-      @data = []
-      @workers_envelopes = {}
-      @log_filename = File.join(@log_folder, "master-#{Process.pid}.log")
-      @sorted_log_filename = File.join(@log_folder, "master-#{Process.pid}_sorted.log")
-
-      FileUtils.mkdir_p(@log_folder)
-      FileUtils.touch(@log_filename)
-    end
-
-    # Start Eventloop
-    #
-    def run
-      EM.run do
-        # Init socket
-        master_socket
-
-        # Init flushing timer
-        flush
-      end
-    end
-
-    # Stop Eventloop
-    #
-    def stop
-      EM.stop
-    end
-
-    # Store data in log array till flush
-    #
-    def map(key, message)
-      @log << "#{key}#{@delimiter}#{message}"
-    end
-
-    # Send data back to worker.
-    # Last item in data is last unfinished session,
-    #   so till the end of file reading we don't send it
-    #
-    def reduce(envelope)
-      if @data.size >= 2 
-        data = @data.shift
-        data = data.flatten
-        master_socket.send_reply(data, envelope)
-      elsif @reduce_stop
-        data = @data.shift
-        data = data.flatten  if data
-        master_socket.send_reply(data, envelope)
-      else
-        EM.add_timer(1) do
-          reduce(envelope)
-        end
-      end
-    end
-
-    # Openning log file for read/write
-    #
-    def log_file
-      @log_file ||= begin
-        File.open(@log_filename, "w+")
-      end
-    end
-
-    # Openning sorted log for reading
-    #
-    def sorted_log_file
-      @sorted_log_file ||= begin
-        File.open(@sorted_log_filename, "r")
-      end
-    end
-
-    # Flushing data to disk once per FLUSH_TIMEOUT seconds
-    #
-    def flush
-      if @log.any?
-        log_file << @log*"\n" << "\n"
-        log_file.flush
-        @log.clear
-      end
-
-      EM.add_timer(FLUSH_TIMEOUT) do
-        flush
-      end
-    end
-
-    # Sorting log.
-    # Linux sort is the fastest way to sort big file.
-    # Deleting original log after sort.
-    #
-    def sort
-      `sort #{@log_filename} -o #{@sorted_log_filename}`
-      FileUtils.rm(@log_filename)
-      @log_file = nil
-    end
-
-    # Start reducing part.
-    # First, flushing rest of log to disk.
-    # Then sort data.
-    # Then start to read/group data
-    #
-    def reduce!
-      flush
-      sort
-
-      iter = sorted_log_file.each_line
-      group iter
-    end
-
-    # Reading sorted data and grouping by key.
-    # If queue (@data) is growing faster then workers grad data we pause reading file.
-    #
-    def group(iter)
-      if @data.size >= GROUP_MAX
-        EM.add_timer(GROUP_TIMEOUT){ group(iter) }
-      else
-        GROUP_LINES.times do
-          line = iter.next.chomp
-          key, msg = line.split(@delimiter)
-
-          last = @data.last
-          if last && last[0] == key
-            last[1] << msg
-          else
-            @data << [key, [msg]]
-          end
-        end
-
-        EM.next_tick{ group(iter) }
-      end
-    rescue StopIteration => e
-      FileUtils.rm(@sorted_log_filename)
-      @sorted_log_file = nil
-      @reduce_stop = true
-    end
-
-    # Initializing and binding socket
-    #
-    def master_socket
-      @master_socket ||= begin
-        sock = MapReduce::Socket::Master.new self, @workers
-        sock.bind @socket_addr
-        sock
-      end
-    end
-  end
-end

data/lib/map_reduce/worker.rb

@@ -1,144 +0,0 @@
-# MapReduce Worker make two jobs:
-#   First, it maps (emits) all data to masters;
-#   Second, it reduces data returned form master;
-# After reducing he is ready to map data again.
-#
-module MapReduce
-  class Worker
-
-    # Valid options:
-    #   * masters - socket addresses of masters, 
-    #       default is 'ipc:///dev/shm/master.sock'
-    #   * type - connection type:
-    #     ** :em - Eventmachine with callbacks (default)
-    #     ** :sync - Synchronous type on Fibers
-    #
-    def initialize(opts = {})
-      @socket_addrs = opts[:masters] || [::MapReduce::DEFAULT_SOCKET]
-
-      @type = opts[:type] ||= :em
-      @socket_class = case @type
-      when :em
-        require File.expand_path("../socket/worker_em", __FILE__)
-        MapReduce::Socket::WorkerEm
-      when :sync
-        require File.expand_path("../socket/worker_sync", __FILE__)
-        MapReduce::Socket::WorkerSync
-      else
-        fail "Wrong Connection type. Choose :em or :sync, not #{opts[:type]}"
-      end
-    end
-
-    # Sends key and value to master through socket.
-    # Key can't be nil.
-    #
-    def emit(key, value, &blk)
-      fail "Key can't be nil"  if key.nil?
-
-      sock = pick_map_socket(key)
-      sock.send_request(["map", key, value], &blk)
-    end
-    alias :map :emit
-
-    # Explicitly stop MAP phase.
-    # Master will wait till all workers will send "map_finished" message.
-    #
-    def map_finished(&blk)
-      all = master_sockets.size
-      resp = 0
-
-      master_sockets.each do |sock, h|
-        sock.send_request(["map_finished"]) do |msg|
-          socket_state(sock, :reduce)
-          blk.call(["ok"])  if block_given? && (resp+=1) == all
-        end
-      end
-      ["ok"]
-    end
-
-    # Reduce operation.
-    # Sends request to all masters.
-    # If master returns nil it means that he is already empty:
-    #  nothing to reduce.
-    # Reducing till any socket returns data.
-    # If nothing to reduce, we return nil to client.
-    #
-    def reduce(&blk)
-      if @type == :em
-        em_reduce(&blk)
-      else
-        sync_reduce(&blk)
-      end
-    end
-
-    def sync_reduce(&blk)
-      while sock = random_reduce_socket
-        key, *values = sock.send_request(["reduce"])
-        if key.nil?
-          socket_state(sock, :map)
-        else
-          blk.call(key, values)
-        end
-      end
-    end
-
-    def em_reduce(&blk)
-      sock = random_reduce_socket
-      if sock
-        sock.send_request(["reduce"]) do |message|
-          key, *values = message
-          if key.nil?
-            socket_state(sock, :map)
-          else
-            blk.call(key, values)
-          end
-
-          em_reduce(&blk)
-        end
-      else
-        blk.call([nil])
-      end
-    end
-
-    private
-
-    # Connect to each master.
-    #
-    def master_sockets
-      @master_sockets ||= begin
-        socks = {}
-        @socket_addrs.each_with_index do |addr, i|
-          sock = @socket_class.new
-          sock.connect addr
-          socks[sock] = { state: :map, ind: i }
-        end
-        socks
-      end
-    end
-
-    # Kind of sharding
-    #
-    def pick_map_socket(key)
-      shard = if master_sockets.size > 1
-        Digest::MD5.hexdigest(key.to_s).to_i(16) % master_sockets.size
-      else
-        0
-      end
-      master_sockets.keys[shard]
-    end
-
-    # Take random socket to get reduce message.
-    # Socket should be in :reduce state.
-    #
-    def random_reduce_socket
-      master_sockets.select{ |k,v| v[:state] == :reduce }.keys.sample
-    end
-
-    # Change socket's state to :map when it is empty
-    #   and to :reduce when mapping is finished
-    #
-    def socket_state(sock, state)
-      master_sockets[sock][:state] = state
-    end
-  end
-end