pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/master/fiber_pool.rb

@@ -0,0 +1,69 @@
+module Pwrake
+
+  class FiberPool
+
+    def initialize(max_fiber=2,&block)
+      @new_fiber_block = block
+      @max_fiber = max_fiber
+      @count = 0
+      @fibers = []
+      @idle_fiber = []
+      @q = []
+      @new_fiber_start_time = Time.now-10
+    end
+
+    def enq(x)
+      @q.push(x)
+      @count += 1
+      if @idle_fiber.empty? and @fibers.size < @max_fiber and
+          Time.now - @new_fiber_start_time > 0.1
+        @idle_fiber << new_fiber
+      end
+      f = @idle_fiber.shift
+      f.resume if f
+      @finished
+    end
+
+    def deq
+      while @q.empty?
+        return nil if @finished
+        @idle_fiber.push(Fiber.current)
+        Fiber.yield
+      end
+      @q.shift
+    end
+
+    def count_down
+      @count -= 1
+    end
+
+    def empty?
+      @count == 0
+    end
+
+    def finish
+      @finished = true
+      run
+      while f = @fibers.shift
+        if f.alive?
+          raise RuntimeError,"FiberPool#finish: fiber is still alive."
+        end
+      end
+    end
+
+    def run
+      cond = !@idle_fiber.empty?
+      while f = @idle_fiber.shift
+        f.resume
+      end
+      cond
+    end
+
+    def new_fiber
+      @fibers.push(fb = @new_fiber_block.call(self))
+      fb
+    end
+
+  end
+end
+

data/lib/pwrake/master/idle_cores.rb

@@ -0,0 +1,30 @@
+module Pwrake
+
+  class IdleCores < Hash
+
+    def increase(k,n)
+      if x = self[k]
+        n += x
+      end
+      self[k] = n
+    end
+
+    def decrease(k,n)
+      x = (self[k]||0) - n
+      if x == 0
+        delete(k)
+      elsif x < 0
+        raise "# of cores must be non-negative"
+      else
+        self[k] = x
+      end
+    end
+
+    def max
+      x = 0
+      each{|k,v| x = v if v > x}
+      x
+    end
+
+  end
+end

data/lib/pwrake/master/master.rb

@@ -0,0 +1,345 @@
+module Pwrake
+
+  class Master
+
+    def initialize
+      @runner = Runner.new
+      @hostid_by_taskname = {}
+      @option = Option.new
+      @hdl_set = HandlerSet.new
+      @channel_by_hostid = {}
+      @channels = []
+      @hosts = {}
+      init_logger
+    end
+
+    attr_reader :task_queue
+    attr_reader :option
+    attr_reader :logger
+
+    def init_logger
+      if logdir = @option['LOG_DIR']
+        ::FileUtils.mkdir_p(logdir)
+        logfile = File.join(logdir,@option['LOG_FILE'])
+        @logger = Logger.new(logfile)
+      else
+        if @option['DEBUG']
+          @logger = Logger.new($stderr)
+        else
+          @logger = Logger.new(File::NULL)
+        end
+      end
+
+      if @option['DEBUG']
+        @logger.level = Logger::DEBUG
+      else
+        @logger.level = Logger::INFO
+      end
+    end
+
+    def init(hosts=nil)
+      @option.init
+      TaskWrapper.init_task_logger(@option)
+    end
+
+    def setup_branch_handler(sub_host)
+      if sub_host == "localhost" && /^(n|f)/i !~ ENV['T']
+        hdl = Handler.new(@runner) do |w0,w1,r2|
+          @thread = Thread.new(r2,w0,@option) do |r,w,o|
+            Rake.application.run_branch_in_thread(r,w,o)
+          end
+        end
+      else
+        hdl = Handler.new(@runner) do |w0,w1,r2|
+          dir = File.absolute_path(File.dirname($PROGRAM_NAME))
+          #args = Shellwords.shelljoin(@args)
+          cmd = "ssh -x -T -q #{sub_host} '" +
+            "cd \"#{Dir.pwd}\";"+
+            "PATH=#{dir}:${PATH} exec pwrake_branch'"
+          Log.debug("BranchCommunicator cmd=#{cmd}")
+          #$stderr.puts "BranchCommunicator cmd=#{cmd}"
+          spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
+          w0.close
+          w1.close
+          r2.close
+        end
+        Marshal.dump(@option,hdl.iow)
+        hdl.iow.flush
+        s = hdl.ior.gets
+        if !s or s.chomp != "pwrake_branch start"
+          raise RuntimeError,"pwrake_branch start failed: receive #{s.inspect}"
+        end
+      end
+      hdl.host = sub_host
+      return hdl
+    end
+
+    def signal_trap(sig)
+      case @killed
+      when 0
+        # log writing failed. can't be called from trap context
+        if Rake.application.options.debug
+          $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid}"+
+            " thread=#{Thread.current} ##{@killed})"
+          $stderr.puts caller
+        else
+          $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid}"+
+            " ##{@killed})"
+        end
+        $stderr.puts "Exiting..."
+        @no_more_run = true
+        @failed = true
+        @hdl_set.kill(sig)
+      when 1
+        $stderr.puts "\nOnce more Ctrl-C (SIGINT) for exit."
+      else
+        Kernel.exit(false) # must wait for nomral exit
+      end
+      @killed += 1
+    end
+
+    def setup_branches
+      sum_ncore = 0
+
+      @option.host_map.each do |sub_host, wk_hosts|
+        @hdl_set << hdl = setup_branch_handler(sub_host)
+        @channels << chan = Channel.new(hdl)
+        chan.puts "host_list_begin"
+        wk_hosts.each do |host_info|
+          name = host_info.name
+          ncore = host_info.ncore
+          host_id = host_info.id
+          Log.debug "connecting #{name} ncore=#{ncore} id=#{host_id}"
+          chan.puts "host:#{host_id} #{name} #{ncore}"
+          @channel_by_hostid[host_id] = chan
+          @hosts[host_id] = name
+        end
+        chan.puts "host_list_end"
+
+        while s = chan.gets
+          case s
+          when /^ncore:done$/
+            break
+          when /^ncore:(\d+):(\d+)$/
+            id, ncore = $1.to_i, $2.to_i
+            Log.debug "worker_id=#{id} ncore=#{ncore}"
+            @option.host_map.by_id[id].set_ncore(ncore)
+            sum_ncore += ncore
+          when /^exited$/
+            raise RuntimeError,"Unexpected branch exit"
+          else
+            msg = "#{hdl.host}:#{s.inspect}"
+            raise RuntimeError,"invalid return: #{msg}"
+          end
+        end
+      end
+
+      Log.info "num_cores=#{sum_ncore}"
+      @hosts.each do |id,host|
+        Log.info "#{host} id=#{id} ncore=#{
+          @option.host_map.by_id[id].idle_cores}"
+      end
+      queue_class = Pwrake.const_get(@option.queue_class)
+      @task_queue = queue_class.new(@option.host_map)
+
+      @branch_setup_thread = Thread.new do
+        @channels.each do |chan|
+          s = chan.gets
+          if /^branch_setup:done$/ !~ s
+            raise RuntimeError,"branch_setup failed"
+          end
+        end
+        @killed = 0
+        [:TERM,:INT].each do |sig|
+          Signal.trap(sig) do
+            signal_trap(sig)
+          end
+        end
+      end
+
+    end
+
+    def create_fiber(channels,&blk)
+      channels.each do |chan|
+        fb = Fiber.new(&blk)
+        fb.resume(chan)
+      end
+    end
+
+    def invoke(t, args)
+      @failed = false
+      t.pw_search_tasks(args)
+
+      if @option['GRAPH_PARTITION']
+        setup_postprocess0
+        @task_queue.deq_noaction_task do |tw,hid|
+          tw.preprocess
+          tw.status = "end"
+          @post_pool.enq(tw)
+        end
+        @runner.run
+        @post_pool.finish
+        Log.debug "@post_pool.finish"
+
+        require 'pwrake/misc/mcgp'
+        MCGP.graph_partition(@option.host_map)
+      end
+
+      setup_postprocess1
+      @branch_setup_thread.join
+      send_task_to_idle_core
+      #
+      create_fiber(@channels) do |chan|
+        while s = chan.get_line
+          Log.debug "Master:recv #{s.inspect} from branch[#{chan.handler.host}]"
+          case s
+          when /^task(\w+):(\d*):(.*)$/o
+            status, shell_id, task_name = $1, $2.to_i, $3
+            tw = Rake.application[task_name].wrapper
+            tw.shell_id = shell_id
+            tw.status = status
+            hid = @hostid_by_taskname[task_name]
+            @task_queue.task_end(tw,hid) # @idle_cores.increase(..
+            # check failure
+            if tw.status == "fail"
+              $stderr.puts %[task "#{tw.name}" failed.]
+              if !@failed
+                @failed = true
+                case @option['FAILURE_TERMINATION']
+                when 'kill'
+                  @hdl_set.kill("INT")
+                  @no_more_run = true
+                  $stderr.puts "... Kill running tasks."
+                when 'continue'
+                  $stderr.puts "... Continue runable tasks."
+                else # 'wait'
+                  @no_more_run = true
+                  $stderr.puts "... Wait for running tasks."
+                end
+              end
+              if tw.has_output_file? && File.exist?(tw.name)
+                handle_failed_target(tw.name)
+              end
+            end
+            # postprocess
+            @post_pool.enq(tw) # must be after @no_more_run = true
+            break if @finished
+          when /^exited$/o
+            @exited = true
+            Log.debug "receive #{s.chomp} from branch"
+            break
+          else
+            Log.error "unknown result: #{s.inspect}"
+            $stderr.puts(s)
+          end
+        end
+        Log.debug "Master#invoke: fiber end"
+      end
+      @runner.run
+      @post_pool.finish
+      Log.debug "Master#invoke: end of task=#{t.name}"
+    end
+
+    def send_task_to_idle_core
+      #Log.debug "#{self.class}#send_task_to_idle_core start"
+      count = 0
+      # @idle_cores.decrease(..
+      @task_queue.deq_task do |tw,hid|
+        count += 1
+        @hostid_by_taskname[tw.name] = hid
+        tw.preprocess
+        if tw.has_action?
+          s = "#{hid}:#{tw.task_id}:#{tw.name}"
+          @channel_by_hostid[hid].put_line(s)
+          tw.exec_host = @hosts[hid]
+        else
+          tw.status = "end"
+          @task_queue.task_end(tw,hid) # @idle_cores.increase(..
+          @post_pool.enq(tw)
+        end
+      end
+      if count == 0 && !@task_queue.empty? && @hostid_by_taskname.empty?
+        m="No task was invoked while unexecuted tasks remain"
+        Log.error m
+        raise RuntimeError,m
+      end
+      #Log.debug "#{self.class}#send_task_to_idle_core end time=#{Time.now-tm}"
+    end
+
+    def setup_postprocess
+      i = 0
+      n = @option.max_postprocess_pool
+      @post_pool = FiberPool.new(n) do |pool|
+        postproc = @option.postprocess(@runner)
+        i += 1
+        Log.debug "New postprocess fiber ##{i}"
+        Fiber.new do
+          j = i
+          while tw = pool.deq()
+            Log.debug "postproc##{j} deq=#{tw.name}"
+            loc = postproc.run(tw)
+            tw.postprocess(loc)
+            pool.count_down
+            @hostid_by_taskname.delete(tw.name)
+            break if yield(pool,j)
+          end
+          postproc.close
+          Log.debug "postproc##{j} end"
+        end
+      end
+    end
+
+    def setup_postprocess0
+      setup_postprocess{false}
+    end
+
+    def setup_postprocess1
+      setup_postprocess do |pool,j|
+        #Log.debug "@no_more_run=#{@no_more_run.inspect}"
+        #Log.debug "@task_queue.empty?=#{@task_queue.empty?}"
+        #Log.debug "@hostid_by_taskname=#{@hostid_by_taskname.inspect}"
+        #Log.debug "pool.empty?=#{pool.empty?}"
+        if (@no_more_run || @task_queue.empty?) &&
+            @hostid_by_taskname.empty?
+          Log.debug "postproc##{j} closing @channels=#{@channels.inspect}"
+          @finished = true
+          @channels.each{|ch| ch.finish} # exit
+          true
+        elsif !@no_more_run
+          send_task_to_idle_core
+          false
+        end
+      end
+    end
+
+    def handle_failed_target(name)
+      case @option['FAILED_TARGET']
+        #
+      when /rename/i, NilClass
+        dst = name+"._fail_"
+        ::FileUtils.mv(name,dst)
+        msg = "Rename failed target file '#{name}' to '#{dst}'"
+        $stderr.puts(msg)
+        Log.warn(msg)
+        #
+      when /delete/i
+        ::FileUtils.rm(name)
+        msg = "Delete failed target file '#{name}'"
+        $stderr.puts(msg)
+        Log.warn(msg)
+        #
+      when /leave/i
+      end
+    end
+
+    def finish
+      Log.debug "Master#finish begin"
+      @branch_setup_thread.join
+      @hdl_set.exit unless @exited
+      TaskWrapper.close_task_logger
+      Log.debug "Master#finish end"
+      @failed
+    end
+
+  end
+end

data/lib/pwrake/master/master_application.rb

@@ -0,0 +1,150 @@
+module Pwrake
+
+  # a mixin for managing Rake application.
+  module MasterApplication
+
+    def pwrake_options
+      @role.option
+    end
+
+    def logger
+      @role.logger
+    end
+
+    def task_logger
+      @role.task_logger
+    end
+
+    def task_queue
+      @role.task_queue
+    end
+
+    # Run the Pwrake application.
+    def run
+      standard_exception_handling do
+        init("pwrake")  # <- parse options here
+        @role = @master = Master.new
+        load_rakefile
+        t = Time.now
+        @master.init
+        @master.setup_branches
+        begin
+          Log.debug "init: #{Time.now-t} sec"
+          t = Time.now
+          top_level
+          Log.debug "main: #{Time.now-t} sec"
+          t = Time.now
+        ensure
+          @failed = @master.finish
+          Log.debug "finish: #{Time.now-t} sec"
+          Log.info "pwrake elapsed time: #{Time.now-START_TIME} sec"
+        end
+        Kernel.exit(false) if @failed
+      end
+    end
+
+    def invoke_task(task_string)
+      name, args = parse_task_string(task_string)
+      t = self[name]
+      @master.invoke(t,args)
+    end
+
+    def standard_rake_options
+      opts = super
+      opts.each_with_index do |a,i|
+        if a[0] == '--version'
+          a[3] = lambda { |value|
+            puts "rake, version #{RAKEVERSION}"
+            puts "pwrake, version #{Pwrake::VERSION}"
+            exit
+          }
+        end
+      end
+
+      opts.concat(
+      [
+       ['-F', '--hostfile FILE',
+        "[Pw] Read hostnames from FILE",
+        lambda { |value|
+          options.hostfile = value
+        }
+       ],
+       ['-j', '--jobs [N]',
+        "[Pw] Number of threads at localhost (default: # of processors)",
+        lambda { |value|
+          if value
+            if /^[+-]?\d+$/ =~ value
+              options.num_threads = value.to_i
+            else
+              raise ArgumentError,"Invalid argument for -j: #{value}"
+            end
+          else
+            options.num_threads = 0
+          end
+        }
+       ],
+       ['-L', '--log', '--log-dir [DIRECTORY]', "[Pw] Write log to DIRECTORY",
+        lambda { |value|
+          if value.kind_of? String
+            options.log_dir = value
+          else
+            options.log_dir = ""
+          end
+        }
+       ],
+       ['--ssh-opt', '--ssh-option OPTION', "[Pw] Option passed to SSH",
+        lambda { |value|
+          options.ssh_option = value
+        }
+       ],
+       ['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|gfarm)",
+        lambda { |value|
+          options.filesystem = value
+        }
+       ],
+       ['--gfarm', "[Pw] FILESYSTEM=gfarm",
+        lambda { |value|
+          options.filesystem = "gfarm"
+        }
+       ],
+       ['-A', '--disable-affinity', "[Pw] Turn OFF affinity (AFFINITY=off)",
+        lambda { |value|
+          options.disable_affinity = true
+        }
+       ],
+       ['-S', '--disable-steal', "[Pw] Turn OFF task steal",
+        lambda { |value|
+          options.disable_steal = true
+        }
+       ],
+       ['-d', '--debug',
+        "[Pw] Output Debug messages",
+        lambda { |value|
+          options.debug = true
+        }
+       ],
+       ['--pwrake-conf [FILE]',
+        "[Pw] Pwrake configuation file in YAML",
+        lambda {|value| options.pwrake_conf = value}
+       ],
+       ['--show-conf','--show-config',
+        "[Pw] Show Pwrake configuration options",
+        lambda {|value| options.show_conf = true }
+       ],
+       ['--report LOGDIR',"[Pw] Report workflow statistics from LOGDIR to HTML and exit.",
+        lambda {|value| options.report_dir = value }
+       ],
+       ['--clear-gfarm2fs',"[Pw] Clear gfarm2fs mountpoints left after failure.",
+         lambda { |value|
+           Option.new.clear_gfarm2fs
+           exit
+         }
+       ],
+
+
+      ])
+      opts
+    end
+
+  end
+end