pmux 0.1.0

data/lib/pmux/job.rb

@@ -0,0 +1,101 @@
+require 'forwardable'
+
+module Pmux
+  class Job
+    extend Forwardable
+    def_delegators :@h, :[], :[]=, :delete
+    attr_reader :tasks, :num_t, :num_r
+    attr_reader :taskhash
+    attr_reader :reducers
+
+    def initialize params, files
+      @params = params
+      @files = files
+
+      @task_id = 0
+      @num_r = @params[:num_r] || 0
+      @params[:job_name] ||= %Q{"#{@params[:mapper]}"}
+
+      @taskhash = {}
+      @done_taskhash = {}
+      @tasks = mk_tasks files
+      @num_t = @tasks.size + @num_r
+      @h = {:start_time=>Time.now,
+        :map_tasks=>@tasks.size, :reduce_tasks=>@num_r,
+      }
+    end
+
+    def mk_tasks files
+      job_id = self.id
+      files.map {|file|
+        @task_id += 1
+        @taskhash[@task_id] = {:job_id=>job_id, :task_id=>@task_id,
+          :file=>file,
+          :mapper=>@params[:mapper], #:reducer=>@params[:reducer],
+          :num_r=>@num_r, :ff=>@params[:ff],
+          :separator=>@params[:separator],
+          :hist=>[],
+        }
+      }
+    end
+
+    def mk_reducer_addrs addrs, num_r=nil
+      num_r ||= @num_r
+      step = addrs.size.to_f / num_r
+      @reducers = (0..num_r-1).map {|ind| addrs[step*ind]}
+      @reducers
+    end
+
+    def mk_reduce_tasks
+      pindex = 0
+      job_id = self.id
+      @tasks = reducers.map {|reducer_node_addr|
+        @task_id += 1
+        #task = make_reduce_task pindex, reducer_node_addr
+        task = {:pindex=>pindex, :job_id=>self.id, :task_id=>@task_id,
+          :node_addr=>reducer_node_addr,
+          :reducer=>@params[:reducer],
+          :hist=>[],
+        }
+        @taskhash[@task_id] = task
+        pindex += 1
+        task
+      }
+    end
+
+    def id
+      self.object_id.abs.to_s
+    end
+
+    def get_task_by_id task_id
+      @taskhash[task_id]
+    end
+
+    def delete_task_by_id task_id
+      if (task = @taskhash[task_id])
+        task[:status] = :DONE
+        @done_taskhash[task_id] = task
+        @taskhash.delete task_id
+      end
+    end
+
+    def completed?
+      @taskhash.empty?
+    end
+
+    def to_jlheader
+      h = {:id=>id, :files_first=>@files.first, :tasksize=>@tasks.size,
+        :params=>@params,
+        :start_time=>@h[:start_time],
+        :map_tasks=>@h[:map_tasks], :reduce_tasks=>@h[:reduce_tasks],
+        :storage_name=>@params[:storage_name],
+        :mapper=>@params[:mapper], :reducer=>@params[:reducer],
+        :num_r=>@params[:num_r],
+      }
+    end
+
+    def to_jlfooter
+      h = {:end_time=>@h[:end_time]}
+    end
+  end
+end

data/lib/pmux/joblogger.rb

@@ -0,0 +1,46 @@
+require 'yaml'
+
+module Pmux
+  class Joblogger
+    attr_reader :path
+
+    def initialize dir, job
+      @dir = dir
+      @job = job
+      if dir and File.directory? dir
+        @path = "#{dir}/#{job.id}.yml"
+        @f = open(@path, 'w')
+      end
+    end
+
+    def dump_header
+      dump @job.to_jlheader
+    end
+
+    def dump_footer
+      dump @job.to_jlfooter
+    end
+
+    def dump obj
+      @f.print YAML.dump(obj) if @f
+    end
+
+    def sep
+      return unless @f
+      @f.puts '---'
+      @f.flush
+    end
+
+    def add key, obj
+      return unless @f
+      @f.puts "#{key}:"
+      for k, v in obj
+        @f.puts "  #{k}: #{v.inspect}"
+      end
+    end
+
+    def close
+      @f.close if @f
+    end
+  end
+end

data/lib/pmux/mapper.rb

@@ -0,0 +1,151 @@
+module Pmux
+  class Mapper
+    attr_accessor :tmp_dir
+    attr_accessor :num_r
+    attr_reader :ifbase
+    attr_reader :exitstatus
+
+    def initialize task, tmp_dir, loop=nil
+      @task = task
+      @tmp_dir = tmp_dir
+      @exitstatus = nil
+      job_id, task_id, fusion_id =
+        task.values_at 'job_id', 'task_id', 'fusion_id'
+      @path = task['path']
+      @num_r = task['num_r'] || 0
+      @loop = loop
+      @on_receive = nil
+      @on_success = nil
+      @on_error = nil
+      if false
+      else
+        @ifbase = "#{tmp_dir}/m#{fusion_id or task_id}"
+      end
+    end
+
+    def do_map_task; end
+    def result_size; end
+    def result_body; end
+  end
+
+  class StreamingMapper < Mapper
+    include FixCmdLine
+    CHUNK_SIZE = 8192
+
+    def on_success &block
+      @on_success = block
+    end
+    def on_error &block
+      @on_error = block
+    end
+
+    def do_map_task
+      mapper_cmd = @task['mapper'] || 'cat'
+      err_path = "#{tmp_dir}/.err.#{$$}"
+      err_msg = nil
+      if @num_r <= 1
+        cmd_line = fix_cmd_line mapper_cmd,
+          @path, "#{@ifbase}-0", err_path, tmp_dir
+        Log.debug "system: #{cmd_line}"
+        system cmd_line
+      else # @num_r >= 2
+        partitioner = TextPartitioner.new @ifbase, @num_r,
+          :separator=>@task['separator']
+        cmd_line = fix_cmd_line mapper_cmd, @path, nil, err_path, tmp_dir
+        IO.popen(cmd_line, 'r') {|io|
+          until io.eof?
+            data = io.read CHUNK_SIZE
+            partitioner.emit data
+          end
+        }
+        partitioner.close
+      end
+      @exitstatus = $?.exitstatus
+      if File.size? err_path
+        err_msg = File.read(err_path).chomp!
+        raise RuntimeError, err_msg
+      end
+      if @exitstatus > 1
+        raise RuntimeError, "failed to execute mapper: #{cmd_line}"
+      end
+      @ifbase
+    end
+
+    def do_streaming_map_task
+      mapper_cmd = @task['mapper'] || 'cat'
+      err_path = "#{tmp_dir}/.err.#{object_id}"
+      err_msg = nil
+      if @num_r <= 1
+        cmd_line = fix_cmd_line mapper_cmd,
+          @path, nil, err_path, tmp_dir
+        Log.debug "pipe: #{cmd_line}"
+        pipeio = PipeIO.new cmd_line
+        out = open("#{@ifbase}-0", 'a')
+        pipeio.on_receive {|data| out.write data}
+      else # @num_r >= 2
+        partitioner = TextPartitioner.new @ifbase, @num_r,
+          :separator=>@task['separator']
+        cmd_line = fix_cmd_line mapper_cmd, @path, nil, err_path, tmp_dir
+        pipeio = PipeIO.new cmd_line
+        pipeio.on_receive {|data| partitioner.emit data}
+      end
+      on_success = @on_success
+      on_error = @on_error
+      pipeio.on_close {
+        if out
+          out.close rescue nil
+        end
+        if partitioner
+          partitioner.close
+        end
+        #@exitstatus = $?.exitstatus
+        if File.size? err_path
+          err_msg = File.read(err_path).chomp!
+          e = RuntimeError.new err_msg
+          e.set_backtrace ['mapper']
+          on_error.call e if on_error
+        else
+          on_success.call if on_success
+        end
+      }
+      @loop.attach pipeio
+    end
+
+    def result_size
+      @num_r <= 1 ? File.size?("#{@ifbase}-0") : nil
+    end
+
+    def result_body
+      @num_r <= 1 ? File.read("#{@ifbase}-0") : nil
+    end
+  end
+
+  class TextPartitioner
+    def initialize ifbase, num_r, options={}
+      @ifbase = ifbase
+      @num_r = num_r
+      @ifiles = (0..(num_r-1)).map {|n| open("#{ifbase}-#{n}", 'w')}
+      @rbuf = ''
+      if (sep = options[:separator])
+        @separator_re = Regexp.new sep
+      else
+        @separator_re = /\t/
+      end
+    end
+
+    def emit data
+      @rbuf << data
+      while true
+        break unless @rbuf =~ /\n/
+        line, s = @rbuf.split /^/, 2
+        key, = line.split @separator_re, 2
+        @ifiles[key.hash % @num_r].write line
+        @rbuf.replace(s || '')
+      end
+    end
+
+    def close
+      @ifiles.each {|io| io.close}
+    end
+  end
+end

data/lib/pmux/mros.rb

@@ -0,0 +1,207 @@
+# msgpack-rpc over ssh
+
+require 'net/ssh'
+if require 'msgpack/rpc'
+  require 'msgpack/rpc/transport/unix'
+  MR = MessagePack::RPC
+end
+if defined? Rev
+  Coolio = Rev
+end
+require 'net/scp'
+
+class Net::SSH::Compat
+  class Watcher < Coolio::IOWatcher
+    def initialize ruby_io, flags, fiber, watchers
+      @ruby_io = ruby_io
+      @fiber = fiber
+      @watchers = watchers
+      super ruby_io, flags
+    end
+
+    def on_readable
+      cleanup
+      @fiber.resume([[@ruby_io]])
+    end
+    def on_writable
+      cleanup
+      @fiber.resume([[], [@ruby_io]])
+    end
+    def cleanup
+      for w in @watchers
+        w.detach
+      end
+    end
+  end
+
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize interval, fiber, watchers
+      @fiber = fiber
+      @watchers = watchers
+      super(interval, false)
+    end
+
+    def on_timer
+      for w in @watchers; w.detach; end
+      @fiber.resume(nil)
+    end
+  end
+
+  class <<self
+    @@loop = Coolio::Loop.default
+
+    def coolio_loop
+      @@loop
+    end
+    def coolio_loop=(loop)
+      @@loop = loop
+    end
+
+    alias :io_select0 :io_select
+    def io_select reads, writes=nil, excepts=nil, timeout=nil
+      if timeout and timeout.zero?
+        io_select0 reads, writes, excepts, 0
+      else
+        writes ||= []
+        loop = @@loop
+
+        watchers = []
+        rw = reads & writes
+        for io in (reads | writes)
+          if rw.include? io
+            flag = :rw
+          elsif reads.include? io
+            flag = :r
+          else
+            flag = :w
+          end
+          watcher = Watcher.new io, flag, Fiber.current, watchers
+          watchers.push watcher
+          watcher.attach loop
+        end
+        if timeout
+          watcher = TimerWatcher.new timeout, Fiber.current, watchers
+          watchers.push watcher
+          watcher.attach loop
+        end
+        Fiber.yield
+      end
+    end
+  end
+end
+
+class Net::SSH::Connection::Session
+  attr_accessor :coolio_loop
+
+  def floop wait=nil
+    while true
+      break if closed?
+      #break unless busy?
+      loop wait
+      @coolio_loop.waitings[Fiber.current] = self
+      Fiber.yield
+    end
+  end
+end
+
+module Coolio
+  class Loop
+    attr_reader :waitings
+    def start_ssh *args, &block
+      @waitings ||= {}
+      fiber = Fiber.new {
+        begin
+          ssh = Net::SSH.start *args
+          ssh.coolio_loop = self
+          block.call ssh
+        rescue => e
+          block.call e
+        end
+      }
+      fiber.resume
+    end
+
+    def run
+      raise RuntimeError, "no watchers for this loop" if @watchers.empty?
+
+      @running = true
+      while @running and not @active_watchers.zero?
+        run_once
+        if @waitings and !@waitings.empty?
+          busy_sessions = @waitings.select {|f, s| s.busy?}
+          for fiber, ssh in busy_sessions
+            @waitings.delete fiber
+            fiber.resume
+          end
+        end
+      end
+      @running = false
+    end
+
+    def set_timer(interval, repeating=false, &block)
+      timer = TimerWatcher.new interval, repeating
+      timer.on_timer {
+        block.call
+        timer.detach unless repeating
+      }
+      timer.attach self
+    end
+  end
+end
+
+module MR
+  class PipeTransport
+    def initialize ruby_in, ruby_out, ruby_err
+      @ruby_in = ruby_in
+      @ruby_out = ruby_out
+      @ruby_err = ruby_err
+    end
+
+    def listen server
+      pout = PipeOut.new @ruby_out
+      pin = PipeIn.new @ruby_in, server, pout
+      perr = PipeOut.new @ruby_err
+      server.loop.attach pout
+      server.loop.attach pin
+      server.loop.attach perr
+    end
+
+    def close
+    end
+
+    class PipeOut < Coolio::IO
+      def initialize ruby_io
+        @ruby_io = ruby_io
+        super ruby_io
+      end
+
+      def on_readable
+      end
+
+      def send_data data
+        @ruby_io.write data
+      end
+    end
+
+    class PipeIn < Coolio::IO
+      include MR::MessageReceiver
+
+      def initialize ruby_io, server, pout
+        @ruby_io = ruby_io
+        super ruby_io
+        @server = server
+        @pout = pout
+        @pac = MessagePack::Unpacker.new
+      end
+      def on_read data
+        @pac.feed_each(data) {|obj| on_message obj}
+      end
+      def on_request msgid, method, param
+        @server.on_request @pout, msgid, method, param
+      end
+      def on_close
+        on_request 0, 'quit', []
+      end
+    end
+  end
+end