pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "0.9.9.2"
+  VERSION = "2.0.0"
 end

data/lib/pwrake/worker/executor.rb

@@ -0,0 +1,169 @@
+module Pwrake
+
+  class Executor
+
+    LIST = {}
+    CHARS='0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
+    TLEN=32
+
+    def initialize(dir_class,id,shell_cmd,shell_rc)
+      @id = id
+      @shell_rc = shell_rc
+      @shell_cmd = shell_cmd || ENV['SHELL'] || '/bin/sh'
+      @terminator = ""
+      TLEN.times{ @terminator << CHARS[rand(CHARS.length)] }
+      @out = Writer.instance
+      @log = LogExecutor.instance
+      @queue = Queue.new
+      @dir = dir_class.new
+      @spawn_in, @sh_in = IO.pipe
+      @sh_out, @spawn_out = IO.pipe
+      @sh_err, @spawn_err = IO.pipe
+      LIST[@id] = self
+      @exec_thread = start_exec_thread
+    end
+
+    def execute(cmd)
+      @queue.enq(cmd)
+    end
+
+    def start_exec_thread
+      Thread.new do
+        begin
+          @dir.open
+          @dir.open_messages.each{|m| @log.info(m)}
+          begin
+            @pid = Kernel.spawn(@shell_cmd,
+                                :out=>@spawn_out,
+                                :err=>@spawn_err,
+                                :in=>@spawn_in,
+                                :chdir=>@dir.current)
+            @out.puts "#{@id}:open"
+            @shell_rc.each do |cmd|
+              run_rc(cmd)
+            end
+            while cmd = @queue.deq
+              run(cmd)
+            end
+            @sh_in.puts("exit")
+            @sh_in.flush
+          ensure
+            status = nil
+            begin
+              timeout(5){
+                pid,status = Process.waitpid2(@pid)
+              }
+            rescue
+              @log.info("#{@id}:kill INT sh @pid=#{@pid}")
+              Process.kill("INT",@pid)
+              pid,status = Process.waitpid2(@pid)
+            end
+            @log.info("shell exit status: "+status.inspect)
+          end
+        rescue => exc
+          @out.puts "#{@id}:exc:#{exc}"
+          @log.error exc
+        ensure
+          @dir.close_messages.each{|m| @log.info(m)}
+          @dir.close
+        end
+      end
+    end
+
+    def run(cmd)
+      case cmd
+      when Proc
+        cmd.call
+      when "cd"
+        @dir.cd
+        run_command("cd "+@dir.current)
+        #
+      when /^cd\s+(.*)$/
+        @dir.cd($1)
+        run_command("cd "+@dir.current)
+        #
+      when /^exit\b/
+        close
+        @out.puts "#{@id}:exit"
+        #
+      when String
+        run_command(cmd)
+        #
+      else
+        raise RuntimeError,"invalid cmd: #{cmd.inspect}"
+      end
+    end
+
+    def run_rc(cmd)
+      run_command_main(cmd){|s| @log.info "<"+s if @log}
+    end
+
+    def run_command(cmd)
+      run_command_main(cmd){|s| @out.puts s}
+    end
+
+    def run_command_main(cmd)
+      if /\\$/ =~ cmd  # command line continues
+        @sh_in.puts(cmd)
+        @sh_in.flush
+        return
+      end
+      term = "\necho '#{@terminator}':$? \necho '#{@terminator}' 1>&2"
+      @sh_in.puts(cmd+term)
+      @sh_in.flush
+      status = ""
+      io_set = [@sh_out,@sh_err]
+      loop do
+        io_sel, = IO.select(io_set,nil,nil)
+        for io in io_sel
+          s = io.gets.chomp
+          case io
+          when @sh_out
+            if s[0,TLEN] == @terminator
+              status = s[TLEN+1..-1]
+              io_set.delete(@sh_out)
+            else
+              yield "#{@id}:o:"+s
+            end
+          when @sh_err
+            if s[0,TLEN] == @terminator
+              io_set.delete(@sh_err)
+            else
+              yield "#{@id}:e:"+s
+            end
+          end
+        end
+        break if io_set.empty?
+      end
+      yield "#{@id}:z:#{status}"
+    end
+
+    def close
+      execute(nil)  # threads end
+    end
+
+    def join
+      LIST.delete(@id)
+      @exec_thread.join(15) if @exec_thread
+    end
+
+    def kill(sig)
+      @queue.clear
+      if @pid
+        # kill process group
+        s = `ps ho pid --ppid=#{@pid}`
+        s.each_line do |x|
+          pid = x.to_i
+          Process.kill(sig,pid)
+          @log.warn "Executor(id=#{@id})#kill pid=#{pid} sig=#{sig}"
+        end
+        if s.empty?
+          @log.warn "Executor(id=#{@id})#kill nothing killed"
+        end
+      end
+      @spawn_out.flush
+      @spawn_err.flush
+    end
+
+  end
+end

data/lib/pwrake/worker/gfarm_directory.rb

@@ -0,0 +1,90 @@
+module Pwrake
+
+  class GfarmDirectory < SharedDirectory
+    @@prefix = nil
+    @@work_dir = nil
+    @@log_dir = nil
+    @@gfarm2fs_option = nil
+    @@gfarm2fs_debug = nil
+    @@gfarm2fs_debug_wait = 1
+    @@current_id = 0
+    @@hostname = `hostname`.chomp
+
+    def self.init(opts)
+      @@prefix   = opts[:base_dir]
+      @@work_dir = opts[:work_dir]
+      @@log_dir  = opts[:log_dir]
+      @@gfarm2fs_option = opts[:gfarm2fs_option]
+      @@gfarm2fs_debug = opts[:gfarm2fs_debug]
+      @@gfarm2fs_debug_wait = opts[:gfarm2fs_debug_wait]
+      Dir.chdir(ENV['HOME'])
+    end
+
+    def initialize
+      super
+      @id = @@current_id
+      @@current_id += 1
+      @suffix = "%05d_%03d" % [Process.pid,@id]
+      @gfarm_mountpoint = @@prefix+"_"+@suffix
+    end
+
+    def home_path
+      Pathname.new(@gfarm_mountpoint)
+    end
+
+    def spawn_cmd(cmd)
+      @log.info "spawn_cmd: "+cmd
+      r,w = IO.pipe
+      pid = spawn(cmd,[:out,:err]=>w)
+      w.close
+      pidmy,status = Process.waitpid2(pid)
+      a = []
+      while s = r.gets
+        a << s.chomp
+      end
+      if status.success?
+        msg = a.empty? ? cmd : cmd+" => #{a.join(',')}"
+        @log.info msg
+      else
+        msg = "failed to execute `#{cmd}' => #{a.join(',')}"
+        raise msg
+      end
+      a
+    end
+
+    def open
+      FileUtils.mkdir_p @gfarm_mountpoint
+      path = @log.path
+      begin
+        if @@gfarm2fs_debug && path
+          f = path+("gfarm2fs-"+`hostname`.chomp+"-"+@suffix)
+          spawn_cmd "gfarm2fs #{@@gfarm2fs_option} -d #{@gfarm_mountpoint} > #{f} 2>&1 & sleep #{@@gfarm2fs_debug_wait}"
+        else
+          spawn_cmd "gfarm2fs #{@@gfarm2fs_option} #{@gfarm_mountpoint}"
+        end
+      rescue => exc
+        sleep 1
+        raise exc
+      end
+      super
+    end
+
+    def close
+      super
+      if File.directory? @gfarm_mountpoint
+        begin
+          spawn_cmd "fusermount -u #{@gfarm_mountpoint}"
+        rescue
+        end
+        system "sync"
+        begin
+          FileUtils.rmdir @gfarm_mountpoint
+          @log.info "rmdir #{@gfarm_mountpoint} @#{@@hostname}"
+        rescue
+          @log.error "failed to rmdir #{@gfarm_mountpoint} @#{@@hostname}"
+        end
+      end
+    end
+
+  end
+end

data/lib/pwrake/worker/invoker.rb

@@ -0,0 +1,199 @@
+require "timeout"
+
+module Pwrake
+
+  class Invoker
+
+    def initialize(dir_class, ncore, option)
+      @dir_class = dir_class
+      @option = option
+      @out = Writer.instance # firstly replace $stderr
+      @log = LogExecutor.instance
+      @log.init(@option)
+      @log.open(@dir_class)
+      @out.add_logger(@log)
+      ncore_max = processor_count()
+      if ncore.kind_of?(Integer)
+        if ncore > 0
+          @ncore = ncore
+        else
+          @ncore = ncore_max + ncore
+        end
+        if @ncore <= 0
+          m = "Out of range: ncore=#{ncore.inspect}"
+          @out.puts "ncore:"+m
+          raise ArgumentError,m
+        end
+      elsif ncore.nil?
+        @ncore = ncore_max
+      else
+        m = "Invalid argument: ncore=#{ncore.inspect}"
+        @out.puts "ncore:"+m
+        raise ArgumentError,m
+      end
+      @out.puts "ncore:#{@ncore}"
+      # does NOT exit when writing to broken pipe
+      Signal.trap("PIPE", "SIG_IGN")
+    end
+
+    def get_line
+      begin
+        line = $stdin.gets
+        exit if !line
+        line.chomp!
+        line.strip!
+        @log.info ">#{line}"
+        return line
+      rescue
+        exit
+      end
+    end
+
+    def run
+      setup_option
+      if setup_loop
+        start_heartbeat
+        command_loop
+      end
+    ensure
+      close_all
+    end
+
+    def setup_option
+      @log.info @option.inspect
+      @heartbeat_interval = @option[:heartbeat]
+      @shell_cmd = @option[:shell_command]
+      @shell_rc = @option[:shell_rc] || []
+      (@option[:pass_env]||{}).each do |k,v|
+        ENV[k] = v
+      end
+    end
+
+    def setup_loop
+      while line = get_line
+        case line
+        when /^(\d+):open$/o
+          $1.split.each do |id|
+            Executor.new(@dir_class,id,@shell_cmd,@shell_rc)
+          end
+        when "setup_end"
+          return true
+        else
+          return false if common_line(line)
+        end
+      end
+      false
+    end
+
+    def start_heartbeat
+      if @heartbeat_interval
+        @heartbeat_thread = Thread.new do
+          while true
+            @out.puts "heartbeat"
+            sleep @heartbeat_interval
+          end
+        end
+      end
+    end
+
+    def command_loop
+      while line = get_line
+        case line
+        when /^(\d+):(.*)$/o
+          id,cmd = $1,$2
+          ex = Executor::LIST[id]
+          if ex.nil?
+            if cmd=="exit"
+              @out.puts "#{id}:end"
+              next
+            else
+              ex = Executor.new(@dir_class,id,@shell_cmd,@shell_rc)
+            end
+          end
+          ex.execute(cmd)
+        else
+          break if common_line(line)
+        end
+      end
+    end
+
+    def common_line(line)
+      case line
+      when /^exit$/o
+        return true
+        #
+      when /^kill:(.*)$/o
+        kill_all($1)
+        return false
+        #
+      when /^p$/o
+        puts "Executor::LIST = #{Executor::LIST.inspect}"
+        return false
+        #
+      else
+        msg = "invalid line: #{line}"
+        @log.fatal msg
+        raise RuntimeError,msg
+      end
+    end
+
+    def kill_all(sig)
+      sig = sig.to_i if /^\d+$/o =~ sig
+      @log.warn "worker_killed:signal=#{sig}"
+      Executor::LIST.each{|id,exc| exc.kill(sig)}
+    end
+
+    def close_all
+      @log.info "close_all"
+      @heartbeat_thread.kill if @heartbeat_thread
+      Dir.chdir
+      id_list = Executor::LIST.keys
+      ex_list = Executor::LIST.values
+      ex_list.each{|ex| ex.close}
+      begin
+        ex_list.each{|ex| ex.join}
+      rescue => e
+        @log.error e
+        @log.error e.backtrace.join("\n")
+      end
+      @log.info "worker:end:#{id_list.inspect}"
+      begin
+        timeout(20){@log.close}
+      rescue => e
+        $stdout.puts e
+        $stdout.puts e.backtrace.join("\n")
+      end
+      @out.puts "exited"
+    end
+
+    # from Michael Grosser's parallel
+    # https://github.com/grosser/parallel
+    def processor_count
+      host_os = RbConfig::CONFIG['host_os']
+      case host_os
+      when /linux|cygwin/
+        ncpu = 0
+        open("/proc/cpuinfo").each do |l|
+          ncpu += 1 if /^processor\s+: \d+/=~l
+        end
+        ncpu
+      when /darwin9/
+        `hwprefs cpu_count`.to_i
+      when /darwin/
+        (hwprefs_available? ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
+      when /(open|free)bsd/
+        `sysctl -n hw.ncpu`.to_i
+      when /mswin|mingw/
+        require 'win32ole'
+        wmi = WIN32OLE.connect("winmgmts://")
+        cpu = wmi.ExecQuery("select NumberOfLogicalProcessors from Win32_Processor")
+        cpu.to_enum.first.NumberOfLogicalProcessors
+      when /solaris2/
+        `psrinfo -p`.to_i # physical cpus
+      else
+        raise "Unknown architecture: #{host_os}"
+      end
+    end
+
+  end
+end