pmux 0.1.0

data/lib/pmux/task_queue.rb

@@ -0,0 +1,11 @@
+module Pmux
+  class TaskQueue < Array
+    def inject_tasks tasks
+      self.concat tasks
+    end
+
+    def delete_job job_id
+      delete_if {|item| item[:job_id] == job_id}
+    end
+  end
+end

data/lib/pmux/task_scheduler.rb

@@ -0,0 +1,166 @@
+
+module Pmux
+  class TaskScheduler
+    attr_reader :shipped
+
+    def initialize adapter=nil
+      @adapter = adapter
+      @node_table = adapter
+      @job_table = {}
+      @task_queue = TaskQueue.new
+      @allocated_tasks = {}
+      @shipped = {}
+    end
+
+    def push_job job
+      @job_table[job.id] = job
+      @task_queue.inject_tasks job.tasks
+    end
+
+    def inject_tasks tasks
+      @task_queue.inject_tasks tasks
+    end
+
+    def delete_task_from_job job, task, node_addr=nil
+      node_addr ||= task[:node_addr]
+      remove_allocated_task node_addr, job.id, task[:task_id]
+      job.delete_task_by_id task[:task_id]
+    end
+
+    def process_queue
+      job_map = {}
+      success_count = 0
+      fail_count = 0
+      fail_lim = @node_table.size * 2
+      tmp_queue = []
+
+      task_queue_size = @task_queue.size
+      loop_count = 0
+      while task = @task_queue.shift
+        if task[:pindex] and task[:node_addr]
+          allocated_p = allocate_reduce_task_to_node job_map, task
+        else
+          unless task[:node_addrs]
+            task[:node_addrs] = @adapter.lookup_file task[:file]
+          end
+          allocated_p = allocate_map_task_to_node job_map, task
+        end
+        if allocated_p
+          # success
+          fail_count = 0
+        else
+          # fail
+          tmp_queue.push task
+          fail_count += 1
+        end
+
+        loop_count += 1
+
+        break if loop_count >= task_queue_size
+      end
+      @task_queue.replace tmp_queue + @task_queue
+
+      flush_job_map job_map
+    end
+
+    #
+    def attach_flush_callback &block
+      @flush_callback = block
+    end
+
+    #
+    def flush_job_map job_map
+      for job_id, nt_map in job_map
+        for node_addr, fslots in nt_map
+          for fslot in fslots
+            if fslot.size > 1
+              task_keys =
+                Hash[*(fslot.map {|t| [t[:task_id], t[:path]]}).flatten]
+              task = fslot.first.merge :task_keys=>task_keys
+              @flush_callback.call node_addr, task if @flush_callback
+            else
+              @flush_callback.call node_addr, fslot.first if @flush_callback
+            end
+          end
+        end
+      end
+    end
+
+    def allocate_task_to_node job_map, task, node_addr
+      if (ff = task[:ff]) and !task[:pindex]
+        # task fusion
+        job_id = task[:job_id]
+        if (nt_map = job_map[job_id]) and (fslots = nt_map[node_addr])
+          for fslot in fslots
+            if fslot.size < ff
+              fslot.push task
+              return true
+            end
+          end
+        end
+      end
+
+      #if @node_table.allocate_task node_addr, task
+      if allocate_task_to_slot node_addr, task
+        # success
+        job_id = task[:job_id]
+        nt_map = (job_map[job_id] ||= {})
+
+        #task = task.dup #???
+        task[:alloc_time] = Time.now
+        (nt_map[node_addr] ||= []).push [task]
+
+        return true
+      end
+      return false
+    end
+
+    #
+    def allocate_task_to_slot node_addr, task
+      if (node = @node_table[node_addr]) and @shipped[node_addr]
+        slot = (@allocated_tasks[node_addr] ||= [])
+        num_workers = node[:num_workers] || 2
+        if slot.size >= num_workers
+          return false
+        else
+          return false if slot.include? task
+          task[:node_addr] = node_addr
+          slot.push task
+          return true
+        end
+      end
+    end
+
+    #
+    def remove_allocated_task node_addr, job_id, task_id
+      if (slot = @allocated_tasks[node_addr])
+        slot.delete_if {|t| t[:job_id] == job_id and t[:task_id] == task_id}
+      end
+    end
+
+    #
+    def allocate_map_task_to_node job_map, task
+      node_addrs = task[:node_addrs].dup
+      for node_addr, path in node_addrs
+        next unless @node_table[node_addr]
+        next unless path
+        task[:path] = path
+        if allocate_task_to_node job_map, task, node_addr
+          task[:node_addrs].delete [node_addr, path]
+          return true
+        end
+      end
+      return false
+    end
+
+    #
+    def allocate_reduce_task_to_node job_map, task
+      return false unless (node_addr = task[:node_addr])
+      if allocate_task_to_node job_map, task, node_addr
+        return true
+      else
+        return false
+      end
+    end
+  end
+end

data/lib/pmux/util_daemon.rb

@@ -0,0 +1,18 @@
+module Process
+  class << self
+    unless method_defined?(:daemon)
+      def daemon(nochdir = nil, noclose = nil)
+        exit!(0) if Process.fork
+        Process.setsid
+        exit!(0) if Process.fork
+        Dir.chdir("/") unless nochdir
+        unless noclose
+          STDIN.reopen("/dev/null", "r")
+          STDOUT.reopen("/dev/null", "w")
+          STDERR.reopen("/dev/null", "w")
+        end
+        0
+      end
+    end
+  end
+end

data/lib/pmux/util_logger.rb

@@ -0,0 +1,137 @@
+require 'logger'
+
+module Log
+  IOBUF = {}
+
+  class Formatter
+    Format = "%s %s\n"
+    attr_accessor :datetime_format
+
+    def call severity, time, progname, msg
+      tstr = time.strftime("%Y-%m-%d %H:%M:%S")
+      Format % [tstr, msg2str(msg)]
+    end
+
+    def msg2str(msg)
+      case msg
+      when ::String
+	msg
+      when ::Exception
+	"#{ msg.message } (#{ msg.class })\n" <<
+	  (msg.backtrace || []).join("\n")
+      else
+	msg.inspect
+      end
+    end
+  end
+
+  class << self
+    attr_accessor :logger
+  end
+
+  module_function
+
+  def init logdev, options={}
+    shift_age = options[:log_shift_age] || 8
+    shift_size = options[:log_shift_size] || 100_000_000
+    logger = Logger.new logdev, shift_age, shift_size
+    logger.formatter = Log::Formatter.new
+    logger.level = {'fatal'=>Logger::FATAL, 'error'=>Logger::ERROR,
+      'warn'=>Logger::WARN, 'info'=>Logger::INFO, 'debug'=>Logger::DEBUG,
+    }[(options[:log_level] || '').downcase] || Logger::INFO
+    Log.logger = logger
+  end
+
+  def open logdev
+    if logdev == '-'
+      logdev = STDOUT
+    end
+    init logdev
+  end
+
+  def close grp=nil
+    if @logger
+      @logger.close
+    else
+      if grp
+	if (io = IOBUF[grp])
+	  io.close
+	end
+      else
+	for io in IOBUF.values
+	  io.close
+	end
+      end
+    end
+  end
+
+  def write grp, str, severity=nil
+    if @logger
+      if severity.kind_of? Symbol
+        severity = @logger.class.const_get severity
+      else
+        severity ||= @logger.class::INFO
+      end
+      begin
+        logger.add severity, str.chomp
+      rescue Logger::Error
+        disable
+      end
+    else
+      io = IOBUF[grp]
+      if io
+	if io.kind_of? File
+	  #mtime = io.mtime
+          tstr = Time.now.strftime("%Y-%m-%d %H:%M:%S ")
+          gstr = (grp == :default) ? '' : "#{grp}: "
+          io.print tstr + gstr + str
+        else
+          io.print str
+          io.print "\n" if severity #XXX
+        end
+      end
+    end
+  end
+
+  def puts(*args)
+    if args.size >= 2 and args[0].kind_of?(Symbol)
+      grp = args.shift
+    else
+      grp = :default
+    end
+    str = args.join('').chomp + "\n"
+    self.write grp, str
+  end
+
+  def pp(*args)
+    if args.size >= 2 and args[0].kind_of?(Symbol)
+      self.puts args[0], args[1..-1].map {|e| e.inspect}.join(', ')
+    else
+      self.puts args.map {|e| e.inspect}.join(', ')
+    end
+  end
+
+  def debug(*args, &block)
+    write :default, args.first, :DEBUG
+  end
+  def info(*args, &block)
+    write :default, args.first, :INFO
+  end
+  def warn(*args, &block)
+    write :default, args.first, :WARN
+  end
+  def error(*args, &block)
+    write :default, args.first, :ERROR
+  end
+  def fatal(*args, &block)
+    write :default, args.first, :FATAL
+  end
+
+  def null(*args, &block)
+  end
+
+  def disable
+    @logger = nil
+    IOBUF.clear
+  end
+end

data/lib/pmux/version.rb

@@ -0,0 +1,3 @@
+module Pmux
+  VERSION = "0.1.0"
+end

data/lib/pmux/worker.rb

@@ -0,0 +1,91 @@
+require 'timeout'
+
+module Pmux
+  class Worker
+    def initialize options, client=nil
+      @options = options
+      unless (@client = client)
+        timeout(3) {
+          @client = init_client(options[:sock_path])
+          @client.timeout = 3600
+        }
+      end
+    end
+
+    def init_client sock_path
+      transport = MR::UNIXTransport.new
+      MR::Client.new transport, sock_path
+    end
+
+    def run
+      Log.debug "W#{$$}: run"
+      while true
+        task = @client.call :get_task
+        result = exec_task task
+        @client.call :put_result, result
+      end
+      Log.debug "W#{$$}: end"
+    rescue MR::TimeoutError
+      Log.debug "W#{$$}: request timed out"
+    rescue MR::TransportError
+      Log.debug "W#{$$}: transport closed"
+    end
+
+    def exec_task task
+      start_time = Time.now
+      if task['task_keys']
+        error_ids = []
+        fusion_id = task['task_id']
+        for task_id, file in task['task_keys']
+          ntask = task.merge 'fusion_id'=>fusion_id,
+            'task_id'=>task_id, 'path'=>file
+          result = do_one_task ntask
+          result.update :task_id=>fusion_id, :task_keys=>task['task_keys']
+        end
+      else
+        result = do_one_task task
+      end
+      result[:welapse] = Time.now - start_time
+      result
+    end
+
+    def do_one_task task
+      job_id, task_id, node_addr =
+        task.values_at 'job_id', 'task_id', 'node_addr'
+      num_r = task['num_r'].to_i
+      result = {:job_id=>job_id, :task_id=>task_id, :node_addr=>node_addr}
+      exception = nil
+      error_level = nil
+      if @options and (otmp_dir = @options[:tmp_dir])
+        tmp_dir = "#{otmp_dir}/#{job_id}"
+      else
+        tmp_dir = "/var/tmp/#{job_id}"
+      end
+      begin
+        if task['pindex']
+          # reduce task
+          result[:reduce] = true
+          reducer = StreamingReducer.new task, tmp_dir
+          result[:output_path] = reducer.do_reduce_task
+        else
+          # map task
+          result[:map] = true
+          mapper = StreamingMapper.new task, tmp_dir
+          result[:ifbase] = mapper.do_map_task
+          if num_r.zero?
+            if (rsize = mapper.result_size)
+              result[:result_body] = mapper.result_body if rsize < 1024
+            else
+              result[:result_body] = ''
+            end
+          end
+        end
+      rescue StandardError => e
+        #q $!, $@[0] if $test
+        result.update :error=>e.class.to_s,
+          :error_message=>e.message, :backtrace=>e.backtrace[0]
+      end
+      result
+    end
+  end
+end

data/lib/pmux/writer.rb

@@ -0,0 +1,19 @@
+module Pmux
+  class Writer
+    CHUNK_SIZE = 8192
+
+    def write; end
+    def finish; end
+  end
+
+  class STDOUTWriter < Writer
+    def write path
+      open(path) {|f|
+        until f.eof?
+          data = f.read(CHUNK_SIZE)
+          STDOUT.write data
+        end
+      }
+    end
+  end
+end

data/lib/pmux.rb

@@ -0,0 +1,27 @@
+File.umask 0022
+ENV['LC_ALL'] = 'C'
+Encoding.default_external = 'ascii-8bit' if RUBY_VERSION > '1.9'
+
+require 'pmux/version'
+require 'pmux/q'
+require 'pmux/util_logger'
+require 'pmux/fiber18'
+require 'pmux/plugin'
+require 'pmux/mros'
+require 'pmux/pipeio'
+require 'pmux/storage_adapter'
+require 'pmux/job'
+require 'pmux/joblogger'
+require 'pmux/fixcmd'
+require 'pmux/mapper'
+require 'pmux/reducer'
+require 'pmux/worker'
+require 'pmux/handler'
+require 'pmux/writer'
+require 'pmux/gatherer'
+require 'pmux/task_queue'
+require 'pmux/task_scheduler'
+require 'pmux/task_dispatcher'
+require 'pmux/multi_session'
+require 'pmux/cleaner'
+require 'pmux/application'

data/pmux.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "pmux/version"
+
+Gem::Specification.new do |s|
+  s.name        = "pmux"
+  s.version     = Pmux::VERSION
+  s.authors     = ["maebashi"]
+  s.homepage    = ""
+  s.summary     = %q{lightweight file-based MapReduce system}
+  s.description = %q{lightweight file-based MapReduce system}
+
+  s.rubyforge_project = "pmux"
+
+  s.files         = `git ls-files`.split("\n").select {|e| /^tmp/!~e}
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+
+  s.add_runtime_dependency "msgpack-rpc"
+  s.add_runtime_dependency "net-scp"
+end