pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb}

@@ -1,6 +1,6 @@
 module Pwrake
 
-  class Profiler
+  class ShellProfiler
 
     HEADER_FOR_PROFILE =
       %w[exec_id task_id task_name command
@@ -13,31 +13,27 @@ module Pwrake
 
     def initialize
       @lock = Mutex.new
-      @separator = ","
-      @re_escape = /\s#{Regexp.escape(@separator)}/
       @gnu_time = false
       @id = 0
       @io = nil
     end
 
-    attr_accessor :separator, :gnu_time
-
     def open(file,gnu_time=false,plot=false)
       @file = file
       @gnu_time = gnu_time
       @plot = plot
       @lock.synchronize do
         @io.close if @io != nil
-        @io = File.open(file,"w")
+        @io = CSV.open(file,"w")
       end
       _puts table_header
       t = Time.now
-      profile(nil,'pwrake_profile_start',t,t)
+      profile(nil,nil,'pwrake_profile_start',t,t)
     end
 
     def close
       t = Time.now
-      profile(nil,'pwrake_profile_end',t,t)
+      profile(nil,nil,'pwrake_profile_end',t,t)
       @lock.synchronize do
         @io.close if @io != nil
         @io = nil
@@ -59,9 +55,10 @@ module Pwrake
       if @gnu_time
         a += HEADER_FOR_GNU_TIME
       end
-      a.join(@separator)
+      a
     end
 
+=begin
     def command(cmd,terminator)
       if @gnu_time
         if /\*|\?|\{|\}|\[|\]|<|>|\(|\)|\~|\&|\||\\|\$|;|`|\n/ =~ cmd
@@ -75,6 +72,7 @@ module Pwrake
         "#{cmd}\necho '#{terminator}':$? "
       end
     end #`
+=end
 
     def format_time(t)
       #t.utc.strftime("%F %T.%L")
@@ -85,35 +83,38 @@ module Pwrake
       t.strftime("%F %T.%L")
     end
 
-    def profile(task, cmd, start_time, end_time, host="", status=nil)
-      status = "" if status.nil?
+    def profile(task_id, task_name, cmd, start_time, end_time, host=nil, status=nil)
       id = @lock.synchronize do
         id = @id
         @id += 1
         id
       end
       if @io
-        if task.kind_of? Rake::Task
-          tname = task.name.inspect
-          task_id = task.task_id
-        else
-          tname = ""
-          task_id = ""
-        end
-        host = '"'+host+'"' if @re_escape =~ host
-        _puts [id, task_id, tname, cmd.inspect,
+        _puts [ id, task_id, task_name, cmd,
                format_time(start_time),
                format_time(end_time),
                "%.3f" % (end_time-start_time),
-               host, status].join(@separator)
+               host, status ]
       end
-      if status==""
+      case status
+      when ""
         1
-      elsif @gnu_time
-        /^([^,]*),/ =~ status
-        Integer($1)
-      else
-        Integer(status)
+      when Integer
+        status
+      when String
+        if @gnu_time
+          if /^([^,]*),/ =~ status
+            Integer($1)
+          else
+            status
+          end
+        else
+          if /^\d+$/ =~ status
+            Integer(status)
+          else
+            status
+          end
+        end
       end
     end
 

data/lib/pwrake/branch/worker_communicator.rb

@@ -0,0 +1,104 @@
+module Pwrake
+
+  class WorkerCommunicator
+
+    HOST2ID = {}
+    RE_ID='\d+'
+    attr_reader :id, :host, :ncore, :handler, :channel
+
+    def self.read_worker_progs(option)
+      d = File.dirname(__FILE__)+'/../worker/'
+      code = ""
+      option.worker_progs.each do |f|
+        code << IO.read(d+f+'.rb')
+      end
+      code
+    end
+
+    def initialize(id,host,ncore,runner,option)
+      @id = id
+      @host = host
+      HOST2ID[@host] = @id
+      @ncore = @n_total_core = ncore
+      #
+      @runner = runner
+      @worker_progs = option.worker_progs
+      @option = option.worker_option
+      if hb = @option[:heartbeat]
+        @heartbeat_timeout = hb + 15
+      end
+    end
+
+    def setup_connection(worker_code)
+      rb_cmd = "ruby -e 'eval ARGF.read(#{worker_code.size})'"
+      if ['localhost','localhost.localdomain','127.0.0.1'].include? @host
+        cmd = "cd; #{rb_cmd}"
+      else
+        cmd = "ssh -x -T -q #{@option[:ssh_option]} #{@host} \"#{rb_cmd}\""
+      end
+      Log.debug cmd
+      @handler = Handler.new(@runner) do |w0,w1,r2|
+        @pid = spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
+        w0.close
+        w1.close
+        r2.close
+      end
+      @handler.host = @host
+      iow = @handler.iow
+      iow.write(worker_code)
+      Marshal.dump(@ncore,iow)
+      Marshal.dump(@option,iow)
+      @channel = Channel.new(@handler)
+    end
+
+    def close
+      if !@closed
+        @closed = true
+        @handler.close
+      end
+    end
+
+    def set_ncore(ncore)
+      @ncore = ncore
+    end
+
+    def ncore_proc(s)
+      if /^ncore:(.*)$/ =~ s
+        a = $1
+        Log.debug "#{a} @#{@host}"
+        if /^(\d+)$/ =~ a
+          set_ncore($1.to_i)
+          return false
+        else
+          raise ArgumentError,"WorkerCommunicator#ncore_proc: s=#{s.inspect}"
+        end
+      else
+        return common_line(s)
+      end
+    end
+
+    def common_line(s)
+      Log.debug "WorkerCommunicator#common_line(#{s.inspect}) id=#{@id} host=#{@host}"
+      case s
+      when /^heartbeat$/
+        @runner.heartbeat(@handler.ior)
+      when /^exited$/
+        return false
+      when /^log:(.*)$/
+        Log.info "worker(#{host})>#{$1}"
+      else
+        Log.warn "worker(#{host}) out> #{s.inspect}"
+      end
+      true
+    end
+
+    def start_default_fiber
+      Fiber.new do
+        while common_line(@channel.get_line)
+        end
+        Log.debug "WorkerCommunicator: end of default fiber"
+      end.resume
+    end
+
+  end
+end

data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb}

@@ -1,5 +1,3 @@
-require 'pwrake/gfwhere_pool'
-
 module Pwrake
 
   module GfarmPath
@@ -101,6 +99,7 @@ module Pwrake
       nil
     end
 
+=begin
     def gfwhere(list)
       system "sync"
       result = {}
@@ -137,104 +136,7 @@ module Pwrake
       end
       result
     end
-
-  end
-
-
-  class GfarmShell < Shell
-
-    @@core_id = {}
-    @@prefix = "pwrake_#{ENV['USER']}"
-
-    def initialize(host,opt={})
-      super(host,opt)
-      @single_mp = @option[:single_mp]
-      @basedir   = @option[:basedir]
-      @prefix    = @option[:prefix] || @@prefix
-      @work_dir  = @option[:work_dir]
-
-      @core_id = @@core_id[host] || 0
-      @@core_id[host] = @core_id + 1
-
-      if @single_mp
-        @remote_mountpoint = "#{@basedir}/#{@prefix}_00"
-      else
-        @remote_mountpoint = "#{@basedir}/#{@prefix}_%02d" % @core_id
-      end
-    end
-
-    def start
-      Log.debug "--- mountpoint=#{@remote_mountpoint}"
-      open(system_cmd)
-      cd
-      if not _system "test -d #{@remote_mountpoint}"
-        _system "mkdir -p #{@remote_mountpoint}" or die
-      else
-        lines = _backquote("sync; mount")
-        if /#{@remote_mountpoint} (?:type )?(\S+)/om =~ lines
-          _system "sync; fusermount -u #{@remote_mountpoint}"
-          _system "sync"
-        end
-      end
-      subdir = GfarmPath.subdir
-      if ["/","",nil].include?(subdir)
-        _system "gfarm2fs #{@remote_mountpoint}"
-      else
-        _system "gfarm2fs -o modules=subdir,subdir=#{subdir} #{@remote_mountpoint}"
-      end
-      path = ENV['PATH'].gsub( /#{GfarmPath.mountpoint}/, @remote_mountpoint )
-      _system "export PATH=#{path}" or die
-      cd_work_dir
-    end
-
-    def close
-      if @remote_mountpoint
-        cd
-        _system "fusermount -u #{@remote_mountpoint}"
-        _system "rmdir #{@remote_mountpoint}"
-      end
-      super
-      self
-    end
-
-    def cd_work_dir
-      # modify local work_dir -> remote work_dir
-      dir = Pathname.new(@remote_mountpoint) + GfarmPath.pwd
-      cd dir
-    end
-
-  end
-
-
-  class GfarmPostprocess
-
-    def initialize
-      max = Pwrake.application.pwrake_options['MAX_GFWHERE_WORKER']
-      @gfwhere_pool = WorkerPool.new(GfwhereWorker,max)
-    end
-
-    def postprocess(t)
-      if t.kind_of?(Rake::FileTask) && t.location.empty?
-        t.location = @gfwhere_pool.work(t.name)
-      end
-    end
-
-    def postprocess_bulk(tasks)
-      list = []
-      tasks.each do |t|
-       list << t.name if t.kind_of? Rake::FileTask
-      end
-      if !list.empty?
-       Log.info "-- after_check: size=#{list.size} #{list.inspect}"
-       gfwhere_result = GfarmPath.gfwhere(list)
-       tasks.each do |t|
-         if t.kind_of? Rake::FileTask
-           t.location = gfwhere_result[GfarmPath.local_to_fs(t.name)]
-         end
-       end
-       #puts "'#{self.name}' exist? => #{File.exist?(self.name)} loc => #{loc}"
-      end
-    end
+=end
 
   end
 end

data/lib/pwrake/gfarm/gfarm_postprocess.rb

@@ -0,0 +1,53 @@
+module Pwrake
+
+  class GfarmPostprocess
+
+    def initialize(runner)
+      @io = IO.popen('gfwhere-pipe','r+')
+      @io.sync = true
+      @hdl = Handler.new(runner,@io,@io)
+      @chan = Channel.new(@hdl)
+    end
+
+    def run(task_wrap)
+      if !task_wrap.is_file_task?
+        return []
+      end
+      filename = task_wrap.name
+      begin
+        @hdl.iow.puts(filename)
+        @hdl.iow.flush
+      rescue Errno::EPIPE
+        Log.warn "GfarmPostprocess#run: Errno::EPIPE for #{filename}"
+        return []
+      end
+      s = @chan.get_line
+      if s.nil?
+        raise "gfwhere: unexpected end"
+      end
+      s.chomp!
+      if s != filename
+        raise "gfwhere: file=#{filename}, result=#{s}"
+      end
+      while s = @chan.get_line
+        s.chomp!
+        case s
+        when /^gfarm:\/\//
+          next
+        when /^Error:/
+          a = []
+          break
+        else
+          a = s.split(/\s+/)
+          break
+        end
+      end
+      Log.debug "Gfarm file=#{filename} nodes=#{a.join("|")}"
+      a
+    end
+
+    def close
+      @io.close
+    end
+  end
+end

data/lib/pwrake/iomux/channel.rb

@@ -0,0 +1,70 @@
+module Pwrake
+
+  class Channel
+
+    def initialize(handler,id=nil)
+      if @id = id
+        @pre = "#{@id}:"
+      else
+        @pre = ""
+      end
+      if !handler.kind_of?(Handler)
+        raise TypeError, "Argument must be Handler but #{handler.class}"
+      end
+      @handler = handler
+    end
+
+    attr_reader :id, :handler, :fiber
+
+    def ior
+      @handler.ior
+    end
+
+    def run_fiber(*args)
+      if @fiber.nil?
+        m = "Channel#run_fiber: @fiber is nil,"+
+          " args=#{args.inspect} @id=#{@id}"
+        $stderr.puts m
+        Log.debug m
+      else
+        @fiber.resume(*args)
+      end
+    end
+
+    def finish
+      if !@fiber.nil?
+        @fiber.resume(nil)
+      end
+    end
+
+    def get_line
+      @handler.add_channel(self)
+      @fiber = Fiber.current
+      line = Fiber.yield
+      @fiber = nil
+      @handler.delete_channel(self)
+      return line
+    end
+
+    def put_line(line)
+      @handler.put_line "#{@pre}#{line}"
+    end
+
+    def puts(line)
+      @handler.puts "#{@pre}#{line}"
+    end
+
+    def gets
+      if @id
+        raise RuntimeError,"gets is invalid when @id is non-nil"
+      else
+        @handler.gets
+      end
+    end
+
+    def inspect
+      "#<#{self.class} io=#{ior.inspect} id=#{id.inspect}>"
+    end
+  end
+
+end