pwrake 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06576ed3f8e9a5ce1a3a551d325a1ff8f279e00e
-  data.tar.gz: 4303b90a5506a9bb028c63dd8ba344e3ab275c08
+  metadata.gz: d3d87023233f03e030697675ddd3869f47f1784a
+  data.tar.gz: 984fc85baa604dfbd0092e833f193e74c56b5a71
 SHA512:
-  metadata.gz: e07ed43f6036919a6741c5db56e8fce972e0cbdb454ba252300e232000b882617baa123a0b602c53dd43b3e63d47034788aa11658626d35bdd59d976a5155149
-  data.tar.gz: 0da74c52c071489794a83e0eba0be5305c0eaa8e31e477dff4eedbd18cec229f8f5dd1ab66ec78f90a1c1e4170137fddd64e22ec08f133b1f1521b4219ee5791
+  metadata.gz: de7d46e56c7487ad87b70b47c5f6d60b0d8231c16b0f86f40aa3437f4618c949986cb14293b05af853a72573f9f110be6ce665905027673b3ad7c2e38a3e911e
+  data.tar.gz: 767e0e3e4a88dfc64224c4358023bde7761084265783386f981b78d60ecd3c573438929a0e426443be1f21b70ba4db5988c5bd9f198586004d3ac4ca8efcdde4

data/README.md

@@ -1,18 +1,27 @@
 # Pwrake
 
-Parallel workflow extension for Rake
+Parallel Workflow extension for Rake, runs on multicores, clusters, clouds.
 * Author: Masahiro Tanaka
 
-([README in Japanese](https://github.com/masa16/pwrake/wiki/Pwrake.ja)),
-([GitHub Repository](https://github.com/masa16/pwrake))
+([README in Japanese](https://github.com/masa16/pwrake/wiki/Pwrakeとは)),
+([GitHub Repository](https://github.com/masa16/pwrake)),
+([RubyGems](https://rubygems.org/gems/pwrake))
 
 ## Features
 
-* Parallelize all tasks; no need to modify Rakefile, no need to use `multitask`.
-* Tasks are executed in the given number of worker threads.
-* Remote exuecution using SSH.
-* Pwrake is an extension to Rake, not patch to Rake: Rake and Pwrake coexist.
-* High parallel I/O performance using Gfarm file system.
+* Pwrake executes a workflow written in Rakefile in parallel.
+  * The specification of Rakefile is same as Rake.
+  * The tasks which do not have mutual dependencies are automatically executed in parallel.
+  * The `multitask` which is a parallel task definition of Rake is no more necessary.
+* Parallel and distributed execution is possible using a computer cluster which consists of multiple compute nodes.
+  * Cluster settings: SSH login, and the directory sharing using a shared filesystem, e.g., NFS, Gfarm.
+  * Pwrake automatically connects to remote hosts using SSH. You do not need to start a daemon.
+  * Remote host names and the number of cores to use are provided in a hostfile.
+* [Gfarm file system](http://sourceforge.net/projects/gfarm/) utilizes storage of compute nodes. It provides the high-performance parallel I/O.
+  * Parallel I/O access to local stroage of compute nodes enables scalable increase in the I/O performance.
+  * Gfarm schedules a compute node to store an output file, to local storage.
+  * Pwrake schedules a compute node to execute a task, to a node where input files are stored.
+  * Other supports for Gfarm: Automatic mount of the Gfarm file system, etc.
 
 ## Installation
 
@@ -97,7 +106,8 @@ Or, gem install:
         SHELL_COMMAND     default=$SHELL
         SHELL_RC          Run-Command when shell starts
         PASS_ENV          (Array) Environment variables passed to SSH
-        HEARTBEAT         defulat=240 - Hearbeat interval in seconds 
+        HEARTBEAT         defulat=240 - Hearbeat interval in seconds
+        RETRY             default=0 - The number of default task retry
         FAILED_TARGET     rename(default)|delete|leave - Treatment of failed target files
         FAILURE_TERMINATION wait(default)|kill|continue - Behavior of other tasks when a task is failed
         QUEUE_PRIORITY          LIHR(default)|FIFO|LIFO|RANK
@@ -123,10 +133,19 @@ Or, gem install:
 
 Example of Rakefile:
 
-    desc "ncore=4 allow=ourhost*"
-    rule ".o" => ".c" do
-      sh "..."
-    end
+``` ruby
+desc "ncore=4 allow=ourhost*" # desc has no effect on rule in original Rake, but it is used for task property in Pwrake.
+rule ".o" => ".c" do
+  sh "..."
+end
+
+(1..n).each do |i|
+  desc "ncore=2 steal=no" # desc should be inside of loop because it is effective only for the next task.
+  file "task#{i}" do
+    sh "..."
+  end
+end
+```
 
 Properties (The leftmost item is default):
 
@@ -136,6 +155,7 @@ Properties (The leftmost item is default):
     deny=hostname     - Deny this host to execute this task. (accepts wild card)
     order=deny,allow|allow,deny - The order of evaluation.
     steal=yes|no      - Allow task stealing for this task.
+    retry=integer     - The number of retry for this task.
 
 ## Note for Gfarm
 

data/lib/pwrake/branch/branch.rb

@@ -125,7 +125,7 @@ module Pwrake
           # wait for remote shell open
           Fiber.new do
             if !shell.open
-              errors << [comm.host,s]
+              errors << comm.host
             end
             Log.debug "Branch#setup_shells: end of fiber to open shell"
           end.resume

data/lib/pwrake/iomux/handler.rb

@@ -62,7 +62,7 @@ module Pwrake
         if default_channel
           return default_channel.run_fiber(s.chomp)
         else
-          raise "No default_channel"
+          raise "No default_channel, received: #{s.chomp}"
         end
       else
         # End of IO

data/lib/pwrake/master/master.rb

@@ -203,22 +203,24 @@ module Pwrake
             # check failure
             if tw.status == "fail"
               $stderr.puts %[task "#{tw.name}" failed.]
-              if !@failed
-                @failed = true
-                case @option['FAILURE_TERMINATION']
-                when 'kill'
-                  @hdl_set.kill("INT")
-                  @no_more_run = true
-                  $stderr.puts "... Kill running tasks."
-                when 'continue'
-                  $stderr.puts "... Continue runable tasks."
-                else # 'wait'
-                  @no_more_run = true
-                  $stderr.puts "... Wait for running tasks."
+              if !tw.retry
+                if !@failed
+                  @failed = true
+                  case @option['FAILURE_TERMINATION']
+                  when 'kill'
+                    @hdl_set.kill("INT")
+                    @no_more_run = true
+                    $stderr.puts "... Kill running tasks."
+                  when 'continue'
+                    $stderr.puts "... Continue runable tasks."
+                  else # 'wait'
+                    @no_more_run = true
+                    $stderr.puts "... Wait for running tasks."
+                  end
+                end
+                if tw.has_output_file? && File.exist?(tw.name)
+                  handle_failed_target(tw.name)
                 end
-              end
-              if tw.has_output_file? && File.exist?(tw.name)
-                handle_failed_target(tw.name)
               end
             end
             # postprocess

data/lib/pwrake/option/host_map.rb

@@ -122,12 +122,14 @@ module Pwrake
       info_list
     end
 
+    REGEX_RANGE = /\[\[([\w\d]+)-([\w\d]+)\]\]/o
+
     def parse_line(info_list,line)
       line = $1 if /^([^#]*)#/ =~ line
       host, ncore, weight, group = line.split
       if host
-        if /\[\[([\w\d]+)-([\w\d]+)\]\]/o =~ host
-          hosts = ($1..$2).map{|i| host.sub(re,i)}
+        if REGEX_RANGE =~ host
+          hosts = ($1..$2).map{|i| host.sub(REGEX_RANGE,i)}
         else
           hosts = [host]
         end

data/lib/pwrake/option/option.rb

@@ -178,6 +178,7 @@ module Pwrake
         ['NUM_THREADS', proc{|v| v && v.to_i}],
         ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
         ['HEARTBEAT', proc{|v| (v || 240).to_i}],
+        ['RETRY', proc{|v| (v || 0).to_i}],
         ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
         ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
         ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],

data/lib/pwrake/queue/queue_array.rb

@@ -69,8 +69,6 @@ module Pwrake
       (size-1).downto(0) do |i|
         if at(i).acceptable_for(host_info)
           return delete_at(i)
-        #else
-        #  Log.debug "task[#{at(i).name}] is rejected"
         end
       end
       nil

data/lib/pwrake/task/task_property.rb

@@ -3,12 +3,15 @@ module Pwrake
   class TaskProperty
 
     attr_reader :ncore, :exclusive, :allow, :deny, :order_allow_deny,
-      :disable_steal
+      :retry, :disable_steal
 
     def parse_description(description)
       if /\bn_?cores?[=:]\s*([+-]?\d+)/i =~ description
         @ncore = $1.to_i
       end
+      if /\bretry[=:]\s*(\d+)/i =~ description
+        @retry = $1.to_i
+      end
       if /\bexclusive[=:]\s*(\S+)/i =~ description
         if /^(y|t)/i =~ $1
           @exclusive = true

data/lib/pwrake/task/task_wrapper.rb

@@ -27,6 +27,7 @@ module Pwrake
       @executed = false
       @assigned = []
       @exec_host = nil
+      @nretry = @property.retry || Rake.application.pwrake_options["RETRY"] || 0
     end
 
     def_delegators :@task, :name, :actions, :prerequisites, :subsequents
@@ -65,6 +66,19 @@ module Pwrake
       @time_start = Time.now
     end
 
+    def retry
+      if @nretry > 0
+        s="retry task: #{name}"
+        Log.debug(s)
+        $stderr.puts(s)
+        @nretry -= 1
+        Rake.application.task_queue.enq(self)
+        true
+      else
+        false
+      end
+    end
+
     def postprocess(location)
       @executed = true if !@task.actions.empty?
       tm_taskend = Time.now

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

data/lib/pwrake/worker/executor.rb

@@ -32,12 +32,12 @@ module Pwrake
         begin
           @dir.open
           @dir.open_messages.each{|m| @log.info(m)}
+          @pid = Kernel.spawn(@shell_cmd,
+                              :out=>@spawn_out,
+                              :err=>@spawn_err,
+                              :in=>@spawn_in,
+                              :chdir=>@dir.current)
           begin
-            @pid = Kernel.spawn(@shell_cmd,
-                                :out=>@spawn_out,
-                                :err=>@spawn_err,
-                                :in=>@spawn_in,
-                                :chdir=>@dir.current)
             @out.puts "#{@id}:open"
             @shell_rc.each do |cmd|
               run_rc(cmd)
@@ -50,10 +50,11 @@ module Pwrake
           ensure
             status = nil
             begin
-              timeout(5){
+              Timeout.timeout(5){
                 pid,status = Process.waitpid2(@pid)
               }
-            rescue
+            rescue => exc
+              @log.error ([exc.to_s]+exc.backtrace).join("\n")
               @log.info("#{@id}:kill INT sh @pid=#{@pid}")
               Process.kill("INT",@pid)
               pid,status = Process.waitpid2(@pid)
@@ -62,7 +63,7 @@ module Pwrake
           end
         rescue => exc
           @out.puts "#{@id}:exc:#{exc}"
-          @log.error exc
+          @log.error ([exc.to_s]+exc.backtrace).join("\n")
         ensure
           @dir.close_messages.each{|m| @log.info(m)}
           @dir.close

data/lib/pwrake/worker/gfarm_directory.rb

@@ -76,7 +76,7 @@ module Pwrake
           spawn_cmd "fusermount -u #{@gfarm_mountpoint}"
         rescue
         end
-        system "sync"
+        #system "sync"
         begin
           FileUtils.rmdir @gfarm_mountpoint
           @log.info "rmdir #{@gfarm_mountpoint} @#{@@hostname}"

data/lib/pwrake/worker/invoker.rb

@@ -1,5 +1,3 @@
-require "timeout"
-
 module Pwrake
 
   class Invoker
@@ -158,7 +156,7 @@ module Pwrake
       end
       @log.info "worker:end:#{id_list.inspect}"
       begin
-        timeout(20){@log.close}
+        Timeout.timeout(20){@log.close}
       rescue => e
         $stdout.puts e
         $stdout.puts e.backtrace.join("\n")

data/lib/pwrake/worker/worker_main.rb

@@ -1,5 +1,6 @@
 require "thread"
 require "fileutils"
+require "timeout"
 
 ncore = Marshal.load($stdin)
 opts = Marshal.load($stdin)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-11 00:00:00.000000000 Z
+date: 2016-02-02 00:00:00.000000000 Z
 dependencies: []
 description: Parallel workflow extension for Rake
 email:
@@ -24,7 +24,6 @@ files:
 - CHANGES_V2.md
 - Gemfile
 - MIT-LICENSE
-- README
 - README.md
 - Rakefile
 - bin/gfwhere-pipe
@@ -119,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Adding Parallel and Distributed feature to Rake
@@ -141,4 +140,3 @@ test_files:
 - spec/helper.rb
 - spec/hosts
 - spec/pwrake_spec.rb
-has_rdoc: 

data/README

@@ -1,12 +0,0 @@
-Pwrake2 - Parallel Workflow extension for Rake
-                   Masahiro Tanaka 2012-04-14
-
-*** under development ***
-
-* Required package
-
- - Ruby 1.9.3 + Rake 0.9.2.2
-
-* Install
-
- $ ruby setup.rb