pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/iomux/handler.rb

@@ -0,0 +1,124 @@
+module Pwrake
+
+  class Handler
+
+    def initialize(runner,*io,&blk)
+      if !runner.kind_of?(Runner)
+        raise TypeError, "Argument must be Runner but #{runner.class}"
+      end
+      @runner = runner
+      @channel = {}
+      if !io.empty?
+        if !block_given?
+          @ior,@iow,@ioe = *io
+        end
+      else
+        if block_given?
+          @ior,w0 = IO.pipe
+          @ioe,w1 = IO.pipe
+          r2,@iow = IO.pipe
+          yield(w0,w1,r2)
+        end
+      end
+      if @ior.nil? && @iow.nil?
+        raise "fail to initialize IO"
+      end
+    end
+
+    attr_reader :runner, :ior, :iow, :ioe
+    attr_accessor :host
+
+    def add_channel(chan)
+      if !chan.kind_of?(Channel)
+        raise TypeError, "Argument must be Channel but #{chan.class}"
+      end
+      @channel[chan.id] = chan
+      @runner.add_handler(self)
+    end
+
+    def delete_channel(chan)
+      if !chan.kind_of?(Channel)
+        raise TypeError, "Argument must be Channel but #{chan.class}"
+      end
+      @channel.delete(chan.id)
+      if @channel.empty?
+        @runner.delete_handler(self)
+      end
+    end
+
+    def process_line
+      if s = (@ior.eof?) ? nil : @ior.gets
+        default_channel = @channel[nil]
+        if @channel.size > ((default_channel) ? 1 : 0)
+          if /^(\d+):(.*)$/ =~ s
+            ch,line = $1,$2
+            if chan = @channel[ch.to_i]
+              return chan.run_fiber(line)
+            else
+              raise "No channel##{ch}"
+            end
+          end
+        end
+        if default_channel
+          return default_channel.run_fiber(s.chomp)
+        else
+          raise "No default_channel"
+        end
+      else
+        # End of IO
+        @channel.values.each do |chan|
+          if chan.fiber
+            chan.run_fiber(nil)
+          end
+        end
+      end
+    end
+
+    def finish
+      @channel.values.each do |chan|
+        chan.finish
+      end
+    end
+
+# -- writer
+
+    def put_line(line)
+      begin
+        @iow.print line.to_str+"\n"
+        @iow.flush
+        #Log.debug "Handler#put_line #{line.inspect} @iow=#{@iow.inspect}"
+      rescue Errno::EPIPE => e
+        if Rake.application.options.debug
+          $stderr.puts "Errno::EPIPE in #{self.class}.put_line '#{line.chomp}'"
+          $stderr.puts e.backtrace.join("\n")
+        end
+        Log.error "Errno::EPIPE in #{self.class}.put_line '#{line.chomp}'\n"+
+          e.backtrace.join("\n")
+        #raise e
+      end
+    end
+
+    def puts(line)
+      @iow.puts(line)
+    end
+
+    def gets
+      @ior.gets
+    end
+
+    def wait_message(end_msg)
+      if line = @ior.gets
+        line.chomp!
+        m = "Handler#wait_message: #{line} host=#{@host}"
+        if line == end_msg
+          Log.debug m
+        else
+          Log.error m
+        end
+      else
+        Log.error "Handler#wait_message: fail to read @ior"
+      end
+    end
+
+  end
+end

data/lib/pwrake/iomux/handler_set.rb

@@ -0,0 +1,35 @@
+module Pwrake
+
+  class HandlerSet < Array
+
+    def kill(sig)
+      each do |hdl|
+        hdl.iow.puts("kill:#{sig}")
+        hdl.iow.flush
+      end
+    end
+
+    def exit
+      a = []
+      each do |hdl|
+        iow = hdl.iow
+        begin
+          iow.puts "exit"
+          iow.flush
+          a << hdl
+          Log.debug "HandlerSet#exit iow=#{iow.inspect}"
+        rescue Errno::EPIPE => e
+          if Rake.application.options.debug
+            $stderr.puts "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}"
+            $stderr.puts e.backtrace.join("\n")
+          end
+          Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
+            e.backtrace.join("\n")
+        end
+      end
+      a.each{|hdl| hdl.wait_message("exited")}
+      Log.debug "HandlerSet#exit end"
+    end
+
+  end
+end

data/lib/pwrake/iomux/runner.rb

@@ -0,0 +1,62 @@
+module Pwrake
+
+  class TimeoutError < IOError; end
+
+  class Runner
+
+    def initialize(timeout=nil)
+      @timeout = timeout
+      @handler = {}
+      @hb_time = {}
+    end
+
+    attr_reader :handler
+
+    def add_handler(hdl)
+      @handler[hdl.ior] ||= hdl
+    end
+
+    def delete_handler(hdl)
+      @handler.delete(hdl.ior)
+    end
+
+    def run
+      while !(io_set = @handler.keys).empty?
+        sel, = IO.select(io_set,nil,nil,@timeout)
+        if sel.nil?
+          raise TimeoutError,"Timeout (#{@timeout} s) in IO.select"
+        end
+        sel.each do |io|
+          @handler[io].process_line
+        end
+        if @timeout && @hb_earliest
+          if Time.now - @hb_earliest > @timeout
+            io = @hb_time.key(@hb_earliest)
+            raise TimeoutError,"Timeout (#{@timeout}s) "+
+              "in Heartbeat from host=#{get_host(io)}"
+          end
+        end
+      end
+    end
+
+    def finish
+      @handler.each do |io,hdl|
+        hdl.finish
+      end
+    end
+
+    # used to print an error message
+    def get_host(io)
+      hdl = @handler[io]
+      h = hdl.respond_to?(:host) ? hdl.host : nil
+    end
+
+    # called when IO start and receive heartbeat
+    def heartbeat(io)
+      @hb_time[io] = Time.now
+      @hb_earliest = @hb_time.values.min
+      #Log.debug "heartbeat: host=#{get_host(io)}"
+    end
+
+  end
+end

data/lib/pwrake/logger.rb

@@ -1,159 +1,12 @@
 module Pwrake
 
-  LOCK = Mutex.new
-
-  class Logger
-
-    attr_accessor :level
-
-    module Severity
-      # Low-level information, mostly for developers
-      DEBUG = 0
-      INFO = 1
-      WARN = 2
-      ERROR = 3
-      FATAL = 4
-      UNKNOWN = 5
-    end
-    include Severity
-
-    def initialize
-      @level = WARN
-      @out = nil
-      @filename = nil
-      @lock = Mutex.new
-    end
-
-    def open(file)
-      close if @out
-      case file
-      when IO
-        @out = file
-        @filename = nil
-      when String
-        @out = File.open(file,"w")
-        @filename = file
-      else
-        raise "file arg must be IO or String"
-      end
-      @start_time = Time.now
-      info "LogStart=" + fmt_time(@start_time)
-      info "logfile=#{@filename}" if @filename
-    end
-
-    def finish(str, start_time)
-      if @out
-        finish_time = Time.now
-        t1 = Log.fmt_time(start_time)
-        t2 = Log.fmt_time(finish_time)
-        elap = finish_time - start_time
-        info "#{str} : start=#{t1} end=#{t2} elap=#{elap}"
-      end
-    end
-
-    def add(severity, message)
-      if !severity || severity >= @level
-        if @out
-          @lock.synchronize do
-            @out.write(message+"\n")
-          end
-        else
-          LOCK.synchronize do
-            $stderr.write(message+"\n")
-          end
-        end
-      end
-      true
-    end
-    alias log add
-
-    def info(msg)
-      add(INFO, msg)
-    end
-
-    def debug(msg)
-      add(DEBUG, msg)
-    end
-
-    def warn(msg)
-      add(WARN, msg)
-    end
-
-    def fmt_time(t)
-      t.strftime("%Y-%m-%dT%H:%M:%S.%%06d") % t.usec
-    end
-
-    def timer(prefix,*args)
-      Timer.new(prefix,*args)
-    end
-
-    def close
-      finish "LogEnd", @start_time
-      @lock.synchronize do
-        @out.close if @filename
-        @out=nil
-      end
-      @filename=nil
-    end
-
-  end # class Logger
-
-
-  LOGGER = Logger.new
-
   module Log
-    include Logger::Severity
 
     module_function
 
-    def open(file)
-      LOGGER.open(file)
-    end
-
-    def close
-      LOGGER.close
-    end
-
-    def info(s)
-      LOGGER.info(s)
-    end
-
-    def debug(s)
-      LOGGER.debug(s)
-    end
-
-    def warn(s)
-      LOGGER.warn(s)
-    end
-
-    def level
-      LOGGER.level
-    end
-
-    def level=(x)
-      LOGGER.level = x
-    end
-
-    def fmt_time(t)
-      t.strftime("%Y-%m-%dT%H:%M:%S.%%06d") % t.usec
-    end
-
-    def timer(prefix,*args)
-      Timer.new(prefix,*args)
-    end
-
-    def stderr_puts(message)
-      LOCK.synchronize do
-        $stderr.write(message+"\n")
-      end
-    end
-
-    def stdout_puts(message)
-      LOCK.synchronize do
-        $stdout.write(message+"\n")
-      end
+    def method_missing(meth_id,*args)
+      Rake.application.logger.send(meth_id,*args)
     end
 
   end
-
-end # module Pwrake
+end

data/lib/pwrake/master.rb

@@ -1,137 +1,30 @@
-module Pwrake
-
-  def current_shell
-    Thread.current[:shell]
-  end
-
-  def current_shell=(a)
-    Thread.current[:shell] = a
-    Thread.current[:hint] = a.host
-  end
-
-  module_function :current_shell, :current_shell=
-
-
-  class Master
-    include Pwrake::Option
-
-    attr_reader :task_queue
-    attr_reader :finish_queue
-    attr_reader :shell_set
-    attr_reader :filesystem
-    attr_reader :postprocess
-
-    def initialize
-      @started = false
-      @lock = Mutex.new
-      @current_task_id = -1
-    end
-
-    def init
-      init_option    # Pwrake::Option
-      setup_option   # Pwrake::Option
-    end
-
-    def start
-      return if @task_queue
-      timer = Timer.new("start_worker")
-      @finish_queue = Queue.new
-      @task_queue = @queue_class.new(host_list)
-      @shell_set = (1..@num_noaction_threads).map{ NoActionShell.new }
-      host_list.core_list.each_with_index do |h,i|
-        @shell_set << @shell_class.new(h,@shell_opt)
-      end
-      start_threads
-      timer.finish
-    end
-
-    def finish
-      Log.debug "-- Master#finish called"
-      @task_queue.finish if @task_queue
-      @threads.each{|t| t.join } if @threads
-      @counter.print if @counter
-      finish_option   # Pwrake::Option
-    end
-
-    def start_threads
-      Thread.abort_on_exception = true
-      @threads = []
-      @exit_task = {}
-      t_intvl = Pwrake.application.pwrake_options['THREAD_CREATE_INTERVAL']
-      @shell_set.each do |c|
-        tc0 = Time.now
-        @threads << Thread.new(c) do |conn|
-          Pwrake.current_shell = conn
-          t0 = Time.now
-          conn.start
-          t = Time.now - t0
-          Log.info "-- worker[#{conn.id}] connect to #{conn.host}: %.3f sec" % t
-          begin
-            thread_loop(conn)
-          ensure
-            Log.info "-- worker[#{conn.id}] ensure : closing #{conn.host}"
-            conn.finish
-          end
-        end
-        t_sleep = t_intvl - (Time.now - tc0)
-        sleep t_sleep if t_sleep > 0
-      end
-    end
-
-    def thread_loop(conn,last=nil)
-      if last
-        @exit_task[last] = Thread.current
-      end
-      hint = (conn) ? conn.host : nil
-      standard_exception_handling do
-        while true
-	  time_start = Time.now
-	  t = @task_queue.deq(hint)
-	  break if !t
-	  time_deq = Time.now - time_start
-          Log.debug "--- Master#thread_loop deq t=#{t.inspect} time=#{time_deq}sec"
-          t.pw_invoke
-          if th = @exit_task.delete(t)
-            @task_queue.thread_end(th)
-          end
-        end
-      end
-    end
-
-    def task_id_counter
-      @lock.synchronize do
-        @current_task_id += 1
-      end
-    end
-
-    # Provide standard execption handling for the given block.
-    def standard_exception_handling
-      begin
-        yield
-      rescue SystemExit => ex
-        # Exit silently with current status
-        @task_queue.stop
-        raise
-      rescue OptionParser::InvalidOption => ex
-        # Exit silently
-        @task_queue.stop
-        exit(false)
-      rescue Exception => ex
-        # Exit with error message
-        name = "pwrake"
-        $stderr.puts "#{name} aborted!"
-        $stderr.puts ex.message
-        if Rake.application.options.trace
-          $stderr.puts ex.backtrace.join("\n")
-        else
-          $stderr.puts ex.backtrace.find {|str| str =~ /#{@rakefile}/ } || ""
-          $stderr.puts "(See full trace by running task with --trace)"
-        end
-        @task_queue.stop
-        exit(false)
-      end
-    end
-
-  end
-
-end # module Pwrake
+require "fileutils"
+require "pathname"
+require "yaml"
+require "logger"
+require "csv"
+
+require "pwrake/version"
+
+require "pwrake/iomux/runner"
+require "pwrake/iomux/handler"
+require "pwrake/iomux/handler_set"
+require "pwrake/iomux/channel"
+
+require "pwrake/logger"
+require "pwrake/master/master"
+require "pwrake/master/idle_cores"
+require "pwrake/master/master_application"
+require "pwrake/master/fiber_pool"
+require "pwrake/option/option"
+require "pwrake/option/option_filesystem"
+require "pwrake/option/host_map"
+require "pwrake/queue/queue_array"
+require "pwrake/queue/task_queue"
+require "pwrake/queue/locality_aware_queue"
+require "pwrake/queue/no_action_queue"
+require "pwrake/task/task_algorithm"
+require "pwrake/task/task_manager"
+require "pwrake/task/task_wrapper"
+require "pwrake/task/task_property"
+require "pwrake/task/task_rank"