pwrake 0.9.9.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b5f302ea2a00a59015b371e07ddf10f646f06eaa
-  data.tar.gz: 57d72fefb63e1a2b8270adf3b479aaba0add62f4
+  metadata.gz: 06576ed3f8e9a5ce1a3a551d325a1ff8f279e00e
+  data.tar.gz: 4303b90a5506a9bb028c63dd8ba344e3ab275c08
 SHA512:
-  metadata.gz: 4990bd96c5a5bc7e268d26efbe07e0cb297b6a629f63f350b3d595e8b27c0831569fdece5a4070991ca7c7017b9914e934c5ed6a0ca444cc5ca3ef937721d040
-  data.tar.gz: 5b89d6aad26a1f802b1e4626aba67815f9914d3549dbb154864eb927558a58e8a16afcdf06025ffafe3724ae99666b8e4ef4db2bf62a3de57cf7f0678d50d7bb
+  metadata.gz: e07ed43f6036919a6741c5db56e8fce972e0cbdb454ba252300e232000b882617baa123a0b602c53dd43b3e63d47034788aa11658626d35bdd59d976a5155149
+  data.tar.gz: 0da74c52c071489794a83e0eba0be5305c0eaa8e31e477dff4eedbd18cec229f8f5dd1ab66ec78f90a1c1e4170137fddd64e22ec08f133b1f1521b4219ee5791

data/.gitignore

@@ -5,6 +5,8 @@
 .yardoc
 Gemfile.lock
 InstalledFiles
+SetupConfig
+SetupReceipt
 _yardoc
 coverage
 doc/

data/CHANGES_V2.md

@@ -0,0 +1,90 @@
+# Changes from 0.9.9.2 to 2.0.0
+
+## Command line options
+
+Obsolete:
+
+    -L, --logfile [FILE]
+
+New:
+
+    -L, --log, --log-dir [DIRECTORY]
+    --report LOGDIR
+    --clear-gfarm2fs
+
+
+## pwrake_conf.yaml or environment variables
+
+Obsolete:
+
+    LOGFILE
+    TASKLOG
+    PROFILE
+    NUM_NOACTION_THREADS
+    THREAD_CREATE_INTERVAL
+    STEAL_WAIT
+    STEAL_WAIT_MAX
+
+New:
+
+    LOG_DIR
+    LOG_FILE
+    TASK_CSV_FILE
+    COMMAND_CSV_FILE
+    GC_LOG_FILE
+    SHELL_COMMAND
+    SHELL_RC
+    HEARTBEAT
+    FAILURE_TERMINATION
+    SHELL_START_INTERVAL
+    GFARM2FS_OPTION
+    GFARM2FS_DEBUG
+    GFARM2FS_DEBUG_WAIT
+
+## The use of Fiber instead of Thread
+* Every Rake Task runs in parallel under Fiber context, instead of Thread.
+* Fiber context does not switch in task action blocks.
+Instead, it switches in "sh" methods, or outside of task action blocks.
+
+### Example of Fiber behavior
+Rakefile:
+```ruby
+T = (1..4).map do |i|
+  task "task#{i}" do
+    sleep 1             # Ruby's sleep method: no context switch
+  end.name
+end
+
+task :default => T
+```
+
+Result:
+
+    $ time pwrake -j 4
+    
+    real	0m4.294s
+    user	0m0.151s
+    sys     0m0.028s
+
+Rakefile:
+```ruby
+T = (1..4).map do |i|
+  task "task#{i}" do
+    sh "sleep 1"        # sleep processes run in parallel
+  end.name
+end
+
+task :default => T
+```
+
+Result:
+
+    $ time pwrake -j 4
+    sleep 1
+    sleep 1
+    sleep 1
+    sleep 1
+    
+    real	0m1.299s
+    user	0m0.155s
+    sys     0m0.030s

data/{LICENSE.txt → MIT-LICENSE}

@@ -1,6 +1,4 @@
-Copyright (c) 2009-2012 Masahiro TANAKA
-
-MIT License
+Copyright (c) 2009-2012 Masahiro Tanaka
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -20,3 +18,4 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README

@@ -0,0 +1,12 @@
+Pwrake2 - Parallel Workflow extension for Rake
+                   Masahiro Tanaka 2012-04-14
+
+*** under development ***
+
+* Required package
+
+ - Ruby 1.9.3 + Rake 0.9.2.2
+
+* Install
+
+ $ ruby setup.rb

data/README.md

@@ -16,7 +16,7 @@ Parallel workflow extension for Rake
 
 ## Installation
 
-Download source tgz/zip and expand, cd to subdir and install:
+Download source tgz/zip and expand, cd to subdirectory and install:
 
     $ ruby setup.rb
 
@@ -52,71 +52,90 @@ Or, gem install:
 
 ## Options
 
-### Command line option
-
-        -F, --hostfile FILE              [Pw] Read hostnames from FILE
-        -j, --jobs [N]                   [Pw] Number of threads at localhost (default: # of processors)
-        -L, --logfile [FILE]             [Pw] Write log to FILE
-            --ssh-opt, --ssh-option OPTION
-                                         [Pw] Option passed to SSH
-            --filesystem FILESYSTEM      [Pw] Specify FILESYSTEM (nfs|gfarm)
-            --gfarm                      [Pw] FILESYSTEM=gfarm
-        -A, --disable-affinity           [Pw] Turn OFF affinity (AFFINITY=off)
-        -S, --disable-steal              [Pw] Turn OFF task steal
-        -d, --debug                      [Pw] Output Debug messages
-            --pwrake-conf [FILE]         [Pw] Pwrake configuation file in YAML
-            --show-conf, --show-config   [Pw] Show Pwrake configuration options
-            --report LOG                 [Pw] Report profile HTML from LOG and exit.
+### Pwrake command line options (in addition to Rake option)
+
+    -F, --hostfile FILE              [Pw] Read hostnames from FILE
+    -j, --jobs [N]                   [Pw] Number of threads at localhost (default: # of processors)
+    -L, --log, --log-dir [DIRECTORY] [Pw] Write log to DIRECTORY
+        --ssh-opt, --ssh-option OPTION
+                                     [Pw] Option passed to SSH
+        --filesystem FILESYSTEM      [Pw] Specify FILESYSTEM (nfs|gfarm)
+        --gfarm                      [Pw] FILESYSTEM=gfarm
+    -A, --disable-affinity           [Pw] Turn OFF affinity (AFFINITY=off)
+    -S, --disable-steal              [Pw] Turn OFF task steal
+    -d, --debug                      [Pw] Output Debug messages
+        --pwrake-conf [FILE]         [Pw] Pwrake configuation file in YAML
+        --show-conf, --show-config   [Pw] Show Pwrake configuration options
+        --report LOGDIR              [Pw] Report workflow statistics from LOGDIR to HTML and exit.
+        --clear-gfarm2fs             [Pw] Clear gfarm2fs mountpoints left after failure.
 
 ### pwrake_conf.yaml
 
 * If `pwrake_conf.yaml` exists at current directory, Pwrake reads options from it.
 * Example (in YAML form):
 
-        HOSTFILE : hosts
-        LOGFILE : true
-        TASKLOG : true
-        PROFILE : true
-        GNU_TIME : true
-        PLOT_PARALLELISM : true
+        HOSTFILE: hosts
+        LOG_DIR: true
         DISABLE_AFFINITY: true
         DISABLE_STEAL: true
-        FAILED_TARGET : delete
+        FAILED_TARGET: delete
         PASS_ENV :
          - ENV1
          - ENV2
 
 * Option list:
 
-        HOSTFILE, HOSTS   default=false
-        LOGFILE, LOG      default=none, string=filename, true="Pwrake%Y%m%d-%H%M%S_%$.log"
-        TASKLOG           default=none, string=filename, true="Pwrake%Y%m%d-%H%M%S_%$.task"
-        PROFILE           default=none, string=filename, true="Pwrake%Y%m%d-%H%M%S_%$.csv"
+        HOSTFILE, HOSTS   nil(default, localhost)|filename
+        LOG_DIR, LOG      nil(default, No log output)|true(dirname="Pwrake%Y%m%d-%H%M%S")|dirname
+        LOG_FILE          default="pwrake.log"
+        TASK_CSV_FILE     default="task.csv"
+        COMMAND_CSV_FILE  default="command.csv"
+        GC_LOG_FILE       default="gc.log"
         WORK_DIR          default=$PWD
-        FILESYSTEM        default=nil (autodetect)
-        SSH_OPTION        (String) SSH option
+        FILESYSTEM        default(autodetect)|gfarm
+        SSH_OPTION        SSH option
+        SHELL_COMMAND     default=$SHELL
+        SHELL_RC          Run-Command when shell starts
         PASS_ENV          (Array) Environment variables passed to SSH
-        GNU_TIME          If true, obtains PROFILEs using GNU time
-        PLOT_PARALLELISM  If true, plot parallelism using GNUPLOT
-        FAILED_TARGET     ( rename(default) | delete | leave ) failed files
-        QUEUE_PRIORITY          RANK(default), FIFO, LIFO, LIHR
-        NOACTION_QUEUE_PRIORITY FIFO(default), LIFO, RAND
-        NUM_NOACTION_THREADS    default=4 when gfarm, else 1
-        THREAD_CREATE_INTERVAL  default=0.01 (sec)
-        HALT_QUEUE_WHILE_SEARCH true|false
-        GRAPH_PARTITION         true|false
-
-  for Gfarm system:
-
-        DISABLE_AFFINITY  default=false
-        DISABLE_STEAL     default=false
-        STEAL_WAIT        default=0 (sec)
-        STEAL_WAIT_MAX    default=10 (sec)
-         : Wait min(STEAL_WAIT*2**n, STEAL_WAIT_MAX) sec for task steal.
-        GFARM_BASEDIR     default="/tmp"
-        GFARM_PREFIX      default="pwrake_$USER"
-        GFARM_SUBDIR      default='/'
+        HEARTBEAT         defulat=240 - Hearbeat interval in seconds 
+        FAILED_TARGET     rename(default)|delete|leave - Treatment of failed target files
+        FAILURE_TERMINATION wait(default)|kill|continue - Behavior of other tasks when a task is failed
+        QUEUE_PRIORITY          LIHR(default)|FIFO|LIFO|RANK
+        NOACTION_QUEUE_PRIORITY FIFO(default)|LIFO|RAND
+        SHELL_START_INTERVAL    default=0.012 (sec)
+        GRAPH_PARTITION         false(default)|true
+
+* Options for Gfarm system:
+
+        DISABLE_AFFINITY    default=false
+        DISABLE_STEAL       default=false
+        GFARM_BASEDIR       default="/tmp"
+        GFARM_PREFIX        default="pwrake_$USER"
+        GFARM_SUBDIR        default='/'
         MAX_GFWHERE_WORKER  default=8
+        GFARM2FS_OPTION     default=""
+        GFARM2FS_DEBUG      default=false
+        GFARM2FS_DEBUG_WAIT default=1
+
+## Task Properties
+
+* Task properties are specified in `desc` strings above task definition in Rakefile.
+
+Example of Rakefile:
+
+    desc "ncore=4 allow=ourhost*"
+    rule ".o" => ".c" do
+      sh "..."
+    end
+
+Properties (The leftmost item is default):
+
+    ncore=integer     - The number of cores used by this task.
+    exclusive=no|yes  - Exclusively execute this task in a single node.
+    allow=hostname    - Allow this host to execute this task. (accepts wild card)
+    deny=hostname     - Deny this host to execute this task. (accepts wild card)
+    order=deny,allow|allow,deny - The order of evaluation.
+    steal=yes|no      - Allow task stealing for this task.
 
 ## Note for Gfarm
 
@@ -135,11 +154,15 @@ Or, gem install:
          --with-metis-include=/usr/local/include \
          --with-metis-lib=/usr/local/lib
 
+## Current version
+
+* Pwrake version 2.0.0
+
 ## Tested Platform
 
-* Ruby 2.1.4
-* Rake 10.1.0
-* CentOS 6.4
+* Ruby 2.2.2
+* Rake 10.4.2
+* CentOS 6.7
 
 ## Acknowledgment
 

data/bin/gfwhere-pipe

@@ -4,6 +4,7 @@ require 'ffi'
 require 'singleton'
 
 module Gfarm
+
   class GfarmError < StandardError
   end
   GFARM_ERR_NO_ERROR = 0
@@ -31,20 +32,26 @@ module Gfarm
     end
     module_function :find_executable
 
-    LIBGFARM_PATH = nil
-    LIBGFARM_FILE = "libgfarm.so"
-    if LIBGFARM_PATH
-      path = File.join(LIBGFARM_PATH,LIBGFARM_FILE)
-    elsif x = find_executable('gfwhere')
-      d = File.dirname(x)
-      %w[lib64 lib].each do |l|
-        f = File.join(d,"..",l,LIBGFARM_FILE)
-        if File.file?(f)
-          path = f
-          break
-        end
+    if LIBGFARM_PATH = ENV['LIBGFARM_PATH']
+      dirs = LIBGFARM_PATH.split(":")
+    elsif d = find_executable('gfwhere')
+      d = File.dirname(File.dirname(d))
+      dirs = %w[lib64 lib].map{|l| File.join(d,l)}
+    else
+      raise StandardError, "cannot find libgfarm path"
+    end
+    path = nil
+    dirs.each do |d|
+      f = File.join(d,"libgfarm.so*")
+      g = Dir.glob(f)
+      if !g.empty?
+        path = g[0]
+        break
       end
     end
+    if !(path && File.exist?(path))
+      raise StandardError, "cannot find libgfarm"
+    end
     extend ::FFI::Library
     ffi_lib path
     attach_function :gfarm_initialize, [:pointer, :pointer], :int
@@ -142,6 +149,10 @@ module Gfarm
 
 end
 
+[:PIPE,:TERM,:INT].each do |sig|
+  Signal.trap(sig, "EXIT")
+end
+
 gfarm = Gfarm.connection
 
 while path=$stdin.gets

data/bin/pwrake

@@ -1,37 +1,30 @@
-#! /usr/bin/env ruby
-
-#--
-# Copyright (c) 2009-2012  Masahiro TANAKA
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-# sell copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#++
+#!/usr/bin/env ruby
 
 begin
-  require 'rubygems'
-  gem 'rake'
+  require 'rake'
 rescue LoadError
+  require 'rubygems'
+  require 'rake'
 end
 
-require 'rake'
-
-libpath = File.expand_path(File.dirname(__FILE__))+"/../lib"
+libpath = File.absolute_path(File.dirname(__FILE__))+"/../lib"
 $LOAD_PATH.unshift libpath
-require "pwrake"
 
+require "pwrake/master"
+require "pwrake/branch"
+class Rake::Application
+  include Pwrake::BranchApplication
+  prepend Pwrake::MasterApplication
+  prepend Pwrake::TaskManager
+end
+class Rake::Task
+  include Pwrake::TaskAlgorithm
+end
+
+# does NOT exit when writing to broken pipe
+Signal.trap(:PIPE, "IGNORE")
+
+#require "tracer"
+#Tracer.on
+#Thread.abort_on_exception = true
 Rake.application.run

data/bin/pwrake_branch

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'rake'
+rescue LoadError
+  require 'rubygems'
+  require 'rake'
+end
+
+libpath = File.absolute_path(File.dirname(__FILE__))+"/../lib"
+$LOAD_PATH.unshift libpath
+
+require "pwrake/branch"
+class Rake::Application
+  include Pwrake::BranchApplication
+end
+
+#require "tracer"
+#Tracer.on
+
+# does NOT exit when writing to broken pipe
+Signal.trap(:PIPE, "IGNORE")
+
+Rake.application.run_branch($stdin,$stdout)