pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/branch.rb

@@ -0,0 +1,22 @@
+require "logger"
+require 'csv'
+
+require "pwrake/logger"
+
+require "pwrake/iomux/channel"
+require "pwrake/iomux/handler"
+require "pwrake/iomux/handler_set"
+require "pwrake/iomux/runner"
+
+require "pwrake/branch/branch_application"
+require "pwrake/branch/branch"
+require "pwrake/branch/fiber_queue"
+require "pwrake/branch/file_utils"
+
+require 'pwrake/branch/shell_profiler'
+require "pwrake/branch/shell"
+require "pwrake/branch/worker_communicator"
+
+require "pwrake/option/option"
+require "pwrake/option/option_filesystem"
+require "pwrake/option/host_map"

data/lib/pwrake/branch/branch.rb

@@ -0,0 +1,213 @@
+module Pwrake
+
+  class Branch
+
+    def initialize(opts,r,w)
+      #Thread.abort_on_exception = true
+      @option = opts
+      @task_q = {}  # worker_id => FiberQueue.new
+      @shells = []
+      @ior = r
+      @iow = w
+      @runner = Runner.new(@option['HEARTBEAT'])
+      @master_hdl = Handler.new(@runner,@ior,@iow)
+      @master_chan = Channel.new(@master_hdl)
+      @wk_comm = {}
+      @wk_hdl_set = HandlerSet.new
+      @shell_start_interval = @option['SHELL_START_INTERVAL']
+    end
+
+    # Rakefile is loaded after 'init' before 'run'
+
+    def run
+      setup_worker
+      setup_shell
+      setup_fiber
+      setup_master_channel
+      @runner.run
+      Log.debug "Brandh#run end"
+    end
+
+    attr_reader :logger
+
+    def init_logger
+      if dir = @option['LOG_DIR']
+        logfile = File.join(dir,@option['LOG_FILE'])
+        @logger = Logger.new(logfile)
+      else
+        if @option['DEBUG']
+          @logger = Logger.new($stderr)
+        else
+          @logger = Logger.new(File::NULL)
+        end
+      end
+
+      if @option['DEBUG']
+        @logger.level = Logger::DEBUG
+      elsif @option['TRACE']
+        @logger.level = Logger::INFO
+      else
+        @logger.level = Logger::WARN
+      end
+    end
+
+    def setup_worker
+      s = @ior.gets
+      if s.chomp != "host_list_begin"
+        raise "Branch#setup_worker: recv=#{s.chomp} expected=host_list_begin"
+      end
+
+      if dir = @option['LOG_DIR']
+        fn = File.join(dir,@option["COMMAND_CSV_FILE"])
+        Shell.profiler.open(fn,@option['GNU_TIME'],@option['PLOT_PARALLELISM'])
+      end
+
+      worker_code = WorkerCommunicator.read_worker_progs(@option)
+
+      while s = @ior.gets
+        s.chomp!
+        break if s == "host_list_end"
+        if /^host:(\d+) (\S+) ([+-]?\d+)?$/ =~ s
+          id, host, ncore = $1,$2,$3
+          ncore &&= ncore.to_i
+          comm = WorkerCommunicator.new(id,host,ncore,@runner,@option)
+          comm.setup_connection(worker_code)
+          @wk_comm[id] = comm
+          @wk_hdl_set << comm.handler
+          @task_q[id] = FiberQueue.new
+        else
+          raise "Branch#setup_worker: recv=#{s.chomp} expected=host:id hostname ncore"
+        end
+      end
+
+      errors = []
+
+      @wk_comm.values.each do |comm|
+        Fiber.new do
+          while true
+            if s = comm.channel.get_line
+              break unless comm.ncore_proc(s)
+            else
+              errors << comm
+              break
+            end
+          end
+          Log.debug "Branch#setup_worker: fiber end of ncore_proc"
+        end.resume
+      end
+
+      @runner.run
+
+      if !errors.empty?
+        errors.each{|comm| @wk_hdl_set.delete(comm.handler)}
+        hosts = errors.map{|comm| comm.host}.join(",")
+        raise RuntimeError,"Failed to connect to workers: #{hosts}"
+      end
+
+      # ncore
+      @wk_comm.each_value do |comm|
+        # set WorkerChannel#ncore at Master
+        @master_hdl.put_line "ncore:#{comm.id}:#{comm.ncore}"
+      end
+      @master_hdl.put_line "ncore:done"
+    end
+
+    def setup_shell
+      @shells = []
+      errors = []
+      shell_id = 0
+      @wk_comm.each_value do |comm|
+        comm.ncore.times do
+          chan = Channel.new(comm.handler,shell_id)
+          shell_id += 1
+          shell = Shell.new(chan,@task_q[comm.id],@option.worker_option)
+          @shells << shell
+          # wait for remote shell open
+          Fiber.new do
+            if !shell.open
+              errors << [comm.host,s]
+            end
+            Log.debug "Branch#setup_shells: end of fiber to open shell"
+          end.resume
+          sleep @shell_start_interval
+        end
+      end
+
+      @runner.run
+
+      if !errors.empty?
+        raise RuntimeError,"Failed to start workers: #{errors.inspect}"
+      end
+    end
+
+    def setup_fiber
+      # start fibers
+      @shells.each do |shell|
+        shell.create_fiber(@master_hdl).resume
+      end
+      Log.debug "all fiber started"
+
+      @wk_comm.each_value do |comm|
+        #comm.start_default_fiber
+        Fiber.new do
+          while s = comm.channel.get_line
+            break unless comm.common_line(s)
+          end
+          Log.debug "Branch#setup_fiber: end of fiber for default channel"
+        end.resume
+      end
+
+      # setup end
+      @wk_comm.values.each do |comm|
+        comm.handler.put_line "setup_end"
+      end
+
+      @master_hdl.put_line "branch_setup:done"
+      Log.debug "Branch#setup_fiber: setup end"
+    end
+
+    def setup_master_channel
+      Fiber.new do
+        while s = @master_chan.get_line
+          # receive command from main pwrake
+          Log.debug "Branch:recv #{s.inspect} from master"
+          case s
+            #
+          when /^(\d+):(.+)$/o
+            id, tname = $1,$2
+            @task_q[id].enq(tname)
+            #
+          when /^exit$/
+            @task_q.each_value{|q| q.finish}
+            @shells.each{|shell| shell.close}
+            @runner.finish
+            break
+            #
+          when /^kill:(.*)$/o
+            sig = $1
+            kill(sig)
+          else
+            Log.debug "Branch: invalid line from master: #{s}"
+          end
+        end
+        Log.debug "Branch#setup_master_channel: end of fiber for master channel"
+      end.resume
+    end
+
+    def kill(sig="INT")
+      Log.warn "Branch#kill #{sig}"
+      @wk_hdl_set.kill(sig)
+    end
+
+    def finish
+      return if @finished
+      @finished = true
+      Log.debug "Branch#finish: begin"
+      @wk_hdl_set.exit
+      Log.debug "Branch#finish: worker exited"
+      @master_hdl.put_line "exited"
+      Log.debug "Branch#finish: sent 'exited' to master"
+    end
+
+  end # Pwrake::Branch
+end

data/lib/pwrake/branch/branch_application.rb

@@ -0,0 +1,53 @@
+module Pwrake
+
+  # The TaskManager module is a mixin for managing tasks.
+  module BranchApplication
+
+    def logger
+      @branch.logger
+    end
+
+    def run_branch(r,w)
+      #standard_exception_handling do
+        init("pwrake_branch")
+        opts = Marshal.load(r)
+        if !opts.kind_of?(Hash)
+          raise "opts is not a Hash: opts=#{opts.inspect}"
+        end
+        @branch = Branch.new(opts,r,w)
+        @branch.init_logger
+        opts.feedback_options
+        load_rakefile
+        w.puts "pwrake_branch start"
+        w.flush
+        begin
+          @branch.run
+        rescue => e
+          Log.fatal e
+          $stderr.puts e
+          $stderr.puts e.backtrace
+          @branch.kill
+        ensure
+          @branch.finish
+        end
+      #end
+    end
+
+    def run_branch_in_thread(r,w,opts)
+      #standard_exception_handling do
+        @branch = Branch.new(opts,r,w)
+        begin
+          @branch.run
+        rescue => e
+          Log.fatal e
+          $stderr.puts e
+          $stderr.puts e.backtrace
+          @branch.kill
+        ensure
+          @branch.finish
+        end
+      #end
+    end
+
+  end
+end

data/lib/pwrake/branch/fiber_queue.rb

@@ -0,0 +1,36 @@
+require 'fiber'
+
+module Pwrake
+
+  class FiberQueue
+
+    def initialize
+      @q = []
+      @waiter = []
+      @finished = false
+    end
+
+    def enq(x)
+      @q.push(x)
+      f = @waiter.shift
+      f.resume if f
+    end
+
+    def deq
+      while @q.empty?
+        return nil if @finished
+        @waiter.push(Fiber.current)
+        Fiber.yield
+      end
+      return @q.shift
+    end
+
+    def finish
+      @finished = true
+      while f = @waiter.shift
+        f.resume
+      end
+    end
+
+  end
+end

data/lib/pwrake/branch/file_utils.rb

@@ -0,0 +1,101 @@
+module Pwrake
+
+  module FileUtils
+    module_function
+
+    def sh(*cmd, &block)
+      options = (Hash === cmd.last) ? cmd.pop : {}
+      unless block_given?
+        show_command = cmd.join(" ")
+        show_command = show_command[0,42] + "..."
+        block = lambda { |ok, status|
+          ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
+        }
+      end
+      if RakeFileUtils.verbose_flag == :default
+        options[:verbose] = true
+      else
+        options[:verbose] ||= RakeFileUtils.verbose_flag
+      end
+      options[:noop]    ||= RakeFileUtils.nowrite_flag
+      Rake.rake_check_options options, :noop, :verbose
+      Rake.rake_output_message cmd.join(" ") if options[:verbose]
+      #Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
+      unless options[:noop]
+        res,status = Pwrake::FileUtils.pwrake_system(*cmd)
+        block.call(res, status)
+      end
+    end
+
+    def bq(*cmd, &block)
+      options = (Hash === cmd.last) ? cmd.pop : {}
+      unless block_given?
+        show_command = cmd.join(" ")
+        show_command = show_command[0,42] + "..."
+        block = lambda { |ok, status|
+          ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
+        }
+      end
+      if RakeFileUtils.verbose_flag == :default
+        options[:verbose] = true
+      else
+        options[:verbose] ||= RakeFileUtils.verbose_flag
+      end
+      options[:noop]    ||= RakeFileUtils.nowrite_flag
+      Rake.rake_check_options options, :noop, :verbose
+      Rake.rake_output_message cmd.join(" ") if options[:verbose]
+      #Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
+      unless options[:noop]
+        res,status = Pwrake::FileUtils.pwrake_backquote(*cmd)
+        block.call(res, status)
+      end
+      res
+    end
+
+    def pwrake_system(*cmd)
+      cmd_log = cmd.join(" ").inspect
+      #tm = Pwrake::Timer.new("sh",cmd_log)
+
+      conn = Pwrake::Shell.current
+      if conn.kind_of?(Pwrake::Shell)
+        res    = conn.system(*cmd)
+        status = Rake::PseudoStatus.new(conn.status)
+      else
+        res    = system(*cmd)
+        status = $?
+        status = Rake::PseudoStatus.new(1) if !res && status.nil?
+      end
+
+      #tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
+      [res,status]
+    end
+
+    # Pwrake version of backquote command
+    def pwrake_backquote(cmd)
+      cmd_log = cmd.inspect
+      #tm = Pwrake::Timer.new("bq",cmd_log)
+
+      conn = Pwrake::Shell.current
+      if conn.kind_of?(Pwrake::Shell)
+        res    = conn.backquote(*cmd)
+        status = Rake::PseudoStatus.new(conn.status)
+      else
+        res    = `#{cmd}`
+        status = $?
+        status = Rake::PseudoStatus.new(1) if status.nil?
+      end
+
+      #tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
+      [res,status]
+    end
+
+  end # module Pwrake::FileUtils
+end
+
+module Rake
+  module DSL
+    include Pwrake::FileUtils
+    private(*Pwrake::FileUtils.instance_methods(false))
+  end
+end
+self.extend Rake::DSL

data/lib/pwrake/branch/shell.rb

@@ -0,0 +1,231 @@
+module Pwrake
+
+  class DummyMutex
+    def synchronize
+      yield
+    end
+  end
+
+  class Shell
+
+    OPEN_LIST={}
+    BY_FIBER={}
+    @@profiler = ShellProfiler.new
+
+    def self.profiler
+      @@profiler
+    end
+
+    def self.current
+      BY_FIBER[Fiber.current]
+    end
+
+    def initialize(chan,task_q,opt={})
+      @chan = chan
+      @host = chan.handler.host
+      @task_q = task_q
+      @lock = DummyMutex.new
+      @id = chan.id
+      #
+      @option = opt
+      @work_dir = @option[:work_dir] || Dir.pwd
+    end
+
+    attr_reader :id, :host, :status, :profile
+
+    def open
+      if @opened
+        Log.warn "already opened: host=#{@host} id=#{@id}"
+      end
+      _puts("open")
+      if (s = _gets) == "open"
+        OPEN_LIST[__id__] = self
+        @opened = true
+        true
+      else
+        Log.error("Shell#open failed: recieve #{s.inspect}")
+        false
+      end
+    end
+
+    def close
+      if !@opened
+        Log.warn "already closed: host=#{@host} id=#{@id}"
+      end
+      _puts("exit")
+      if (s = _gets) == "exit"
+        OPEN_LIST.delete(__id__)
+        @opened = false
+        true
+      else
+        Log.warn("Shell#close failed: recieve #{s.inspect}")
+        false
+      end
+    end
+
+    def set_current_task(task_id,task_name)
+      @task_id = task_id
+      @task_name = task_name
+    end
+
+    def backquote(*command)
+      command = command.join(' ')
+      @lock.synchronize do
+        a = []
+        _execute(command){|x| a << x}
+        a.join("\n")
+      end
+    end
+
+    def system(*command)
+      command = command.join(' ')
+      @lock.synchronize do
+        _execute(command){|x| print x+"\n"}
+      end
+      @status == 0
+    end
+
+    def cd(dir="")
+      _system("cd #{dir}") or die
+    end
+
+    def die
+      raise "Failed at #{@host}, id=#{@id}, cmd='#{@cmd}'"
+    end
+
+    at_exit {
+      Shell.profiler.close
+    }
+
+    private
+
+    def _puts(s)
+      #Log.debug "Shell#_puts(host=#{@host},id=#{@id}): #{s.inspect}"
+      @chan.put_line(s)
+    end
+
+    def _gets
+      s = @chan.get_line
+      #Log.debug "Shell#_gets(host=#{@host},id=#{@id}): #{s.inspect}"
+      if s.nil?
+        begin
+          raise
+        rescue => e
+          Log.debug e
+        end
+      end
+      s
+    end
+
+    def _system(cmd)
+      @cmd = cmd
+      #raise "@chan is closed" if @chan.closed?
+      @lock.synchronize do
+        _puts(cmd)
+        status = io_read_loop{}
+        Integer(status||1) == 0
+      end
+    end
+
+    def _backquote(cmd)
+      @cmd = cmd
+      #raise "@chan is closed" if @chan.closed?
+      a = []
+      @lock.synchronize do
+        _puts(cmd)
+        status = io_read_loop{|x| a << x}
+      end
+      a.join("\n")
+    end
+
+    def _execute(cmd,quote=nil,&block)
+      @cmd = cmd
+      if !@opened
+        raise "closed"
+      end
+      status = nil
+      start_time = Time.now
+      begin
+        _puts(cmd)
+        @status = io_read_loop(&block)
+      ensure
+        end_time = Time.now
+        @status = @@profiler.profile(@task_id, @task_name, cmd,
+                                     start_time, end_time, host, @status)
+      end
+    end
+
+    def io_read_loop
+      while s = _gets
+        case s
+        when /^(\w+):(.*)$/
+          x = [$1,$2]
+          case x[0]
+          when "o"
+            yield x[1]
+            next
+          when "e"
+            $stderr.print x[1]+"\n"
+            next
+          when "z"
+            # see Executor#status_to_str
+            status = x[1]
+            case status
+            when /^\d+$/
+              status = status.to_i
+            end
+            return status
+          when "err"
+            # see Executor#status_to_str
+            status = x[1]
+            case status
+            when /^\d+$/
+              status = status.to_i
+            end
+            return status
+          end
+        end
+        msg = "Shell#io_read_loop: Invalid result: #{s.inspect}"
+        $stderr.puts(msg)
+        Log.error(msg)
+      end
+    end
+
+    public
+
+    def create_fiber(hdl)
+      if !@opened
+        Log.warn "not opened: host=#{@host} id=#{@id}"
+      end
+      Fiber.new do
+        BY_FIBER[Fiber.current] = self
+        Log.debug "shell start id=#{@id} host=#{@host}"
+        begin
+          while task_str = @task_q.deq
+            #Log.debug "task_str=#{task_str}"
+            if /^(\d+):(.*)$/ =~ task_str
+              task_id, task_name = $1.to_i, $2
+            else
+              raise RuntimeError, "invalid task_str: #{task_str}"
+            end
+            @task_id = task_id
+            @task_name = task_name
+            task = Rake.application[task_name]
+            begin
+              task.execute(task.arguments) if task.needed?
+              result = "taskend:#{@id}:#{task.name}"
+            rescue Exception=>e
+              Rake.application.display_error_message(e)
+              Log.error e
+              result = "taskfail:#{@id}:#{task.name}"
+            end
+            hdl.put_line result
+          end
+        ensure
+          Log.debug "shell id=#{@id} fiber end"
+        end
+      end
+    end
+
+  end
+end