pwrake 2.1.3 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db880d8483f79d6812bd1920557350a7b4311378
-  data.tar.gz: a1d2779a59eb21dd4df2b8ee5e9a00054a50654e
+  metadata.gz: f781cdfe39747649da0fafb2ebb37e579d294483
+  data.tar.gz: 5985be277367367065e9fef64188fc43ec8a2de4
 SHA512:
-  metadata.gz: 4bc21c0d7a5ef317d5d03883ca918e2c2218e0ec28e13167079127820e0c4800923f4d23c6891e4eff5dcdfb010db4b2d6214a6d9eedf5c6c5597a1353467d7a
-  data.tar.gz: b851a5ffb98ddccf77b52bbd5c4339a900d617ce8018e9c68ee665999ac9a70a8b5295be21c88832a8d9f20cd257da81b9b4c91715f487fde0711a60db4e9931
+  metadata.gz: 847964dd1146902e2f6f5988a26188184845e1b7879ed43a311560266e90d33d3c5febe7bb6727cc2003075b540046028a6cbaea6c28451888866d1e43aa1cfe
+  data.tar.gz: 76af1e8e961e9ff22e1c6ca0626a416eb73f2f43a9ce83c0c4b72a311841fd319b6a99c1fbf9594cbf5a4efafe8dc00833dae772550b705837f58e40f48ad69a

data/.gitignore

@@ -30,3 +30,5 @@ tmp
 rhosts
 spec/*/*.dat
 spec/*/*.csv
+spec/*/hoge*
+spec/*/Pwrake*

data/README.md

@@ -3,9 +3,9 @@
 Parallel Workflow extension for Rake, runs on multicores, clusters, clouds.
 * Author: Masahiro Tanaka
 
-([README in Japanese](https://github.com/masa16/pwrake/wiki/Pwrakeとは)),
-([GitHub Repository](https://github.com/masa16/pwrake)),
-([RubyGems](https://rubygems.org/gems/pwrake))
+[README in Japanese](https://github.com/masa16/pwrake/wiki/Pwrakeとは),
+[GitHub Repository](https://github.com/masa16/pwrake),
+[RubyGems](https://rubygems.org/gems/pwrake)
 
 ## Features
 
@@ -14,7 +14,7 @@ Parallel Workflow extension for Rake, runs on multicores, clusters, clouds.
   * The tasks which do not have mutual dependencies are automatically executed in parallel.
   * The `multitask` which is a parallel task definition of Rake is no more necessary.
 * Parallel and distributed execution is possible using a computer cluster which consists of multiple compute nodes.
-  * Cluster settings: SSH login, and the directory sharing using a shared filesystem, e.g., NFS, Gfarm.
+  * Cluster settings: SSH login (or MPI), and the directory sharing using a shared filesystem, e.g., NFS, Gfarm.
   * Pwrake automatically connects to remote hosts using SSH. You do not need to start a daemon.
   * Remote host names and the number of cores to use are provided in a hostfile.
 * [Gfarm file system](http://sourceforge.net/projects/gfarm/) utilizes storage of compute nodes. It provides the high-performance parallel I/O.
@@ -68,7 +68,15 @@ In this case, you need the rehash of command paths:
 
 4. Run `pwrake` with an option `--hostfile` or `-F`:
 
-        $ pwrake --hostfile=hosts
+        $ pwrake -F hosts
+
+### Use MPI to start remote worker
+
+1. Setup MPI on your cluster.
+2. Install [MPipe gem](https://rubygems.org/gems/mpipe). (requires `mpicc`)
+3. Run `pwrake-mpi` command.
+
+        $ pwrake-mpi -F hosts
 
 ## Options
 
@@ -115,8 +123,6 @@ In this case, you need the rehash of command paths:
         WORK_DIR          default=$PWD
         FILESYSTEM        default(autodetect)|gfarm
         SSH_OPTION        SSH option
-        SHELL_COMMAND     default=$SHELL
-        SHELL_RC          Run-Command when shell starts
         PASS_ENV          (Array) Environment variables passed to SSH
         HEARTBEAT         default=240 - Hearbeat interval in seconds
         RETRY             default=1 - The number of retry
@@ -177,7 +183,7 @@ Properties (The leftmost item is default):
 
         gem install ffi
 
-## For Graph Partitioning
+## Scheduling with Graph Partitioning
 
 * Compile and Install METIS 5.1.0 (http://www.cs.umn.edu/~metis/). This requires CMake.
 
@@ -187,15 +193,22 @@ Properties (The leftmost item is default):
          --with-metis-include=/usr/local/include \
          --with-metis-lib=/usr/local/lib
 
+* Option (`pwrake_conf.yaml`):
+
+        GRAPH_PARTITION: true
+
+* See publication: [M. Tanaka and O. Tatebe, “Workflow Scheduling to Minimize Data Movement Using Multi-constraint Graph Partitioning,” in CCGrid 2012](http://ieeexplore.ieee.org/abstract/document/6217406/)
+
 ## Current version
 
-* Pwrake version 2.0.0
+* Pwrake version 2.2.0
 
 ## Tested Platform
 
-* Ruby 2.2.2
-* Rake 10.4.2
-* CentOS 6.7
+
+* Ruby 2.4.0
+* Rake 12.0.0
+* CentOS 7.3
 
 ## Acknowledgment
 

data/bin/pwrake-mpi

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'rake'
+rescue LoadError
+  require 'rubygems'
+  require 'rake'
+end
+
+libpath = File.absolute_path(File.dirname(__FILE__))+"/../lib"
+$LOAD_PATH.unshift libpath
+
+require "pwrake/version"
+require "pwrake/master/master_application"
+require "shellwords"
+
+module Pwrake
+module MasterApplication
+  def run
+    standard_exception_handling do
+      init("pwrake")  # <- parse options here
+      opts = Option.new
+      hosts = opts.host_map.map{|b,a| a.map{|h| h.name}}.flatten
+      if opts['MASTER_IS_FIRST_HOST']
+        [hosts[0],*hosts]
+      else
+        [Socket.gethostname,*hosts]
+      end
+    end
+  end
+end;end
+
+class Rake::Application
+  prepend Pwrake::MasterApplication
+end
+
+hosts = Rake.application.run.join(',')
+args = ARGV.map{|x| Shellwords.escape(x)}.join(" ")
+
+cmd="mpirun -wdir \"$HOME\" -host #{hosts} pwrake-mpi-run \"$PWD\" #{args}"
+exec cmd

data/bin/pwrake-mpi-run

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+libpath = File.absolute_path(File.dirname(__FILE__))+"/../lib"
+$LOAD_PATH.unshift libpath
+
+require "mpipe"
+MPipe.init
+
+wdir = ARGV.shift
+
+if MPipe::Comm.rank == 0
+  Dir.chdir(wdir)
+  require "pwrake/mpi/branch"
+else
+  require "pwrake/mpi/worker"
+end

data/lib/pwrake/branch/branch.rb

@@ -9,6 +9,12 @@ module Pwrake
 
   class Branch
 
+    @@io_class = IO
+
+    def self.io_class=(io_class)
+      @@io_class = io_class
+    end
+
     def initialize(opts,r,w)
       Thread.abort_on_exception = true
       @option = opts
@@ -16,10 +22,17 @@ module Pwrake
       @shells = []
       @ior = r
       @iow = w
-      @selector = NBIO::Selector.new
+      @selector = NBIO::Selector.new(@@io_class)
       @master_rd = NBIO::Reader.new(@selector,@ior)
       @master_wt = NBIO::Writer.new(@selector,@iow)
       @shell_start_interval = @option['SHELL_START_INTERVAL']
+
+      # init_logger
+      Log.set_logger(@option)
+      if dir = @option['LOG_DIR']
+        fn = File.join(dir,@option["COMMAND_CSV_FILE"])
+        Shell.profiler.open(fn,@option['GNU_TIME'],@option['PLOT_PARALLELISM'])
+      end
     end
 
     # Rakefile is loaded after 'init' before 'run'
@@ -33,34 +46,6 @@ module Pwrake
       Log.debug "Branch#run end"
     end
 
-    attr_reader :logger
-
-    def init_logger
-      if dir = @option['LOG_DIR']
-        logfile = File.join(dir,@option['LOG_FILE'])
-        @logger = Logger.new(logfile)
-      else
-        if @option['DEBUG']
-          @logger = Logger.new($stderr)
-        else
-          @logger = Logger.new(File::NULL)
-        end
-      end
-
-      if @option['DEBUG']
-        @logger.level = Logger::DEBUG
-      elsif @option['TRACE']
-        @logger.level = Logger::INFO
-      else
-        @logger.level = Logger::WARN
-      end
-
-      if dir = @option['LOG_DIR']
-        fn = File.join(dir,@option["COMMAND_CSV_FILE"])
-        Shell.profiler.open(fn,@option['GNU_TIME'],@option['PLOT_PARALLELISM'])
-      end
-    end
-
     def setup_worker
       @cs = CommunicatorSet.new(@master_rd,@selector,@option.worker_option)
       @cs.create_communicators
@@ -76,6 +61,9 @@ module Pwrake
         @cs.each_value do |comm|
           # set WorkerChannel#ncore at Master
           @master_wt.put_line "ncore:#{comm.id}:#{comm.ncore}"
+          comm.ipaddr.each do |ipa|
+            @master_wt.put_line "ip:#{comm.id}:#{ipa}"
+          end
         end
         @master_wt.put_line "ncore:done"
       end.resume

data/lib/pwrake/branch/branch_application.rb

@@ -6,51 +6,41 @@ module Pwrake
   # The TaskManager module is a mixin for managing tasks.
   module BranchApplication
 
-    def logger
-      @branch.logger
-    end
-
     def run_branch(r,w)
-      #standard_exception_handling do
-        init("pwrake_branch")
-        opts = Marshal.load(r)
-        if !opts.kind_of?(Hash)
-          raise "opts is not a Hash: opts=#{opts.inspect}"
-        end
-        @branch = Branch.new(opts,r,w)
-        @branch.init_logger
-        opts.feedback_options
-        load_rakefile
-        w.puts "pwrake_branch start"
-        w.flush
-        begin
-          @branch.run
-        rescue => e
-          Log.fatal e
-          $stderr.puts e
-          $stderr.puts e.backtrace
-          @branch.kill
-        ensure
-          @branch.finish
-        end
-      #end
+      init("pwrake_branch")
+      opts = Marshal.load(r)
+      if !opts.kind_of?(Hash)
+        raise "opts is not a Hash: opts=#{opts.inspect}"
+      end
+      @branch = Branch.new(opts,r,w)
+      opts.feedback_options
+      load_rakefile
+      w.puts "pwrake_branch start"
+      w.flush
+      begin
+        @branch.run
+      rescue => e
+        Log.fatal e
+        $stderr.puts e
+        $stderr.puts e.backtrace
+        @branch.kill
+      ensure
+        @branch.finish
+      end
     end
 
     def run_branch_in_thread(r,w,opts)
-      #standard_exception_handling do
-        @branch = Branch.new(opts,r,w)
-        @branch.init_logger
-        begin
-          @branch.run
-        rescue => e
-          Log.fatal e
-          $stderr.puts e
-          $stderr.puts e.backtrace
-          @branch.kill
-        ensure
-          @branch.finish
-        end
-      #end
+      @branch = Branch.new(opts,r,w)
+      begin
+        @branch.run
+      rescue => e
+        Log.fatal e
+        $stderr.puts e
+        $stderr.puts e.backtrace
+        @branch.kill
+      ensure
+        @branch.finish
+      end
     end
 
   end

data/lib/pwrake/branch/communicator.rb

@@ -38,6 +38,7 @@ class Communicator
   attr_reader :id, :host, :ncore, :channel
   attr_reader :reader, :writer, :handler
   attr_reader :shells
+  attr_reader :ipaddr
 
   def initialize(set,id,host,ncore,selector,option)
     @set = set
@@ -47,6 +48,7 @@ class Communicator
     @selector = selector
     @option = option
     @shells = {}
+    @ipaddr = []
   end
 
   def inspect
@@ -58,10 +60,9 @@ class Communicator
     CommChannel.new(@host,i,q,@writer,[@ior,@iow,@ioe])
   end
 
-  def connect(worker_code)
+  def setup_pipe(worker_code)
     rb_cmd = "ruby -e 'eval ARGF.read(#{worker_code.size})'"
-    if ['localhost','localhost.localdomain','127.0.0.1'].include? @host
-    #if /^localhost/ =~ @host
+    if %w[127.0.0.1 ::1].include?(IPSocket.getaddress(@host))
       cmd = rb_cmd
     else
       cmd = "ssh -x -T #{@option[:ssh_option]} #{@host} \"#{rb_cmd}\""
@@ -74,18 +75,33 @@ class Communicator
     w0.close
     w1.close
     r2.close
+    # send worker_code
+    @iow.write(worker_code)
+  end
+
+  def connect(worker_code)
+    setup_pipe(worker_code)
+
+    # send ncore and options
+    opts = Marshal.dump(@option)
+    s = [@ncore||0, opts.size].pack("V2")
+    @iow.write(s)
+    @iow.write(opts)
+
     sel = @set.selector
     @reader = NBIO::MultiReader.new(sel,@ior)
     @rd_err = NBIO::Reader.new(sel,@ioe)
     @writer = NBIO::Writer.new(sel,@iow)
     @handler = NBIO::Handler.new(@reader,@writer,@host)
-    #
-    @writer.write(worker_code)
-    @writer.write(Marshal.dump(@ncore))
-    @writer.write(Marshal.dump(@option))
+
     # read ncore
     while s = @reader.get_line
-      if /^ncore:(.*)$/ =~ s
+      case s
+      when /^ip:(.*)$/
+        a = $1
+        @ipaddr.push(a)
+        Log.debug "ip=#{a} @#{@host}"
+      when /^ncore:(.*)$/
         a = $1
         Log.debug "ncore=#{a} @#{@host}"
         if /^(\d+)$/ =~ a
@@ -134,9 +150,14 @@ class Communicator
     err_out = []
     begin
       finish_shells
-      @handler.exit
-      while s = @rd_err.get_line
-        err_out << s
+      if @handler
+        @handler.exit
+        @handler = nil
+      end
+      if @rd_err
+        while s = @rd_err.get_line
+          err_out << s
+        end
       end
     rescue => e
       m = Log.bt(e)

data/lib/pwrake/branch/communicator_set.rb

@@ -11,10 +11,16 @@ class CommunicatorSet
     @selector = selector
     @option = option
     @communicators = {}
+    @error_host = []
     @initial_communicators = []
     if hb = @option[:heartbeat]
       @heartbeat_timeout = hb + 15
     end
+    init_hosts
+  end
+
+  def init_hosts
+    # for pwrake-mpi
   end
 
   attr_reader :selector

data/lib/pwrake/logger.rb

@@ -4,10 +4,38 @@ module Pwrake
 
   module Log
 
+    @@logger = nil
+
     module_function
 
+    def set_logger(option)
+      unless @@logger
+        if logdir = option['LOG_DIR']
+          ::FileUtils.mkdir_p(logdir)
+          logfile = File.join(logdir, option['LOG_FILE'])
+          @@logger = Logger.new(logfile)
+        else
+          if option['DEBUG']
+            @@logger = Logger.new($stderr)
+          else
+            @@logger = Logger.new(File::NULL)
+          end
+        end
+
+        if option['DEBUG']
+          @@logger.level = Logger::DEBUG
+        else
+          @@logger.level = Logger::INFO
+        end
+
+        at_exit{@@logger.close}
+      end
+    end
+
     def method_missing(meth_id,*args)
-      Rake.application.logger.send(meth_id,*args)
+      if @@logger
+        @@logger.send(meth_id,*args)
+      end
     end
 
     def bt(e)