pwrake 0.9.3

data/.gitignore

@@ -0,0 +1,30 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.config
+*~
+*/*~
+*/*/*~
+*.bak
+*/*.bak
+*/*/*.bak
+.#*
+*/.#*
+*/*/.#*
+rhosts
+spec/*/*.dat
+spec/*/*.csv

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pwrake.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2009-2012 Masahiro TANAKA
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# Pwrake
+
+Parallel workflow extension for Rake
+* Author: Masahiro Tanaka
+
+## Features
+
+* Parallelize all tasks; no need to modify Rakefile, no need to use `multitask`.
+* Tasks are executed in the given number of worker threads.
+* Remote exuecution using SSH.
+* Pwrake is an extension to Rake, not patch to Rake: Rake and Pwrake coexist.
+* High parallel I/O performance using Gfarm file system.
+
+## Installation
+
+Download source tgz/zip and expand, cd to subdir and install:
+
+    $ ruby setup.rb
+
+Or, gem install:
+
+    $ gem install pwrake
+
+## Usage
+
+### Parallel execution using 4 cores at localhost:
+
+    $ pwrake -j 4
+
+### Parallel execution using all cores at localhost:
+
+    $ pwrake -j
+
+### Parallel execution using total 2*2 cores at remote 2 hosts:
+
+1. Share your directory among remote hosts via distributed file system such as NFS, Gfarm.
+2. Allow passphrase-less access via SSH in either way:
+   * Add passphrase-less key generated by `ssh-keygen`.  (Be careful)
+   * Add passphrase using `ssh-add`.
+3. Make `hosts` file in which remote host names and the number of cores are listed:
+
+        $ cat hosts
+        host1 2
+        host2 2
+
+4. Run `pwrake` with an option `--hostfile` or `-F`:
+
+        $ pwrake --hostfile=hosts
+
+## Tested Platform
+
+* Ruby 1.9.3, 2.0.0-preview1
+* Rake 0.9.2.2
+* Fedora 16 / Debian 5.0.7
+
+## Workflow demo
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/pwrake

@@ -0,0 +1,36 @@
+#! /usr/bin/env ruby
+
+#--
+# Copyright (c) 2009-2012  Masahiro TANAKA
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#++
+
+begin
+  require 'rubygems'
+rescue LoadError
+end
+
+require 'rake'
+
+libpath = File.expand_path(File.dirname(__FILE__))+"/../lib"
+$LOAD_PATH.unshift libpath
+require "pwrake"
+
+Rake.application.run

data/lib/pwrake/application.rb

@@ -0,0 +1,187 @@
+module Rake
+  class << self
+    def application
+      @application ||= Pwrake.application
+    end
+  end
+end
+
+
+module Pwrake
+
+  class << self
+    def application
+      @application ||= Application.new
+    end
+  end
+
+  class Application < ::Rake::Application
+
+    def task_queue
+      @master.task_queue
+    end
+
+    def thread_loop(*args)
+      @master.thread_loop(*args)
+    end
+
+    def pwrake_options
+      @master.pwrake_options
+    end
+
+
+    # Run the Pwrake application.
+    def run
+      standard_exception_handling do
+        @master = Master.new
+        init("pwrake")
+        @master.init
+        load_rakefile
+        @master.setup
+        top_level
+      end
+    end
+
+    # Run the top level tasks of a Rake application.
+    def top_level
+      standard_exception_handling do
+        if options.show_tasks
+          display_tasks_and_comments
+        elsif options.show_prereqs
+          display_prerequisites
+        else
+          begin
+            @master.start
+            top_level_tasks.each { |task_name| invoke_task(task_name) }
+          ensure
+            @master.finish
+          end
+        end
+      end
+    end
+
+
+    def standard_rake_options
+      opts = super
+      opts.each_with_index do |a,i|
+        if a[0] == '--version'
+          a[3] = lambda { |value|
+            puts "rake, version #{RAKEVERSION}"
+            puts "pwrake, version #{Pwrake::VERSION}"
+            exit
+          }
+        end
+      end
+
+      opts.concat(
+      [
+       ['-F', '--hostfile FILE',
+        "[Pw] Read hostnames from FILE",
+        lambda { |value|
+          options.hostfile = value
+        }
+       ],
+       ['-j', '--jobs [N]',
+        "[Pw] Number of threads at localhost (default: # of processors)",
+        lambda { |value|
+          if value
+            value = value.to_i
+            if value > 0
+              options.num_threads = value
+            else
+              options.num_threads = x = processor_count + value
+              raise "negative/zero number of threads (#{x})" if x <= 0
+            end
+          else
+            options.num_threads = processor_count
+          end
+        }
+       ],
+       ['-L', '--logfile [FILE]', "[Pw] Write log to FILE",
+        lambda { |value|
+          if value.kind_of? String
+            options.logfile = value
+          else
+            options.logfile = ""
+          end
+        }
+       ],
+       ['--ssh-opt', '--ssh-option OPTION', "[Pw] Option passed to SSH",
+        lambda { |value|
+          options.ssh_option = value
+        }
+       ],
+       ['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|gfarm)",
+        lambda { |value|
+          options.filesystem = value
+        }
+       ],
+       ['--gfarm', "[Pw] FILESYSTEM=gfarm",
+        lambda { |value|
+          options.filesystem = "gfarm"
+        }
+       ],
+       ['-A', '--disable-affinity', "[Pw] Turn OFF affinity (AFFINITY=off)",
+        lambda { |value|
+          options.disable_affinity = true
+        }
+       ],
+       ['-S', '--disable-steal', "[Pw] Turn OFF task steal",
+        lambda { |value|
+          options.disable_steal = true
+        }
+       ],
+       ['-d', '--debug',
+        "[Pw] Output Debug messages",
+        lambda { |value|
+          options.debug = true
+        }
+       ],
+       ['--pwrake-conf [FILE]',
+        "[Pw] Pwrake configuation file in YAML",
+        lambda {|value| options.pwrake_conf = value}
+       ],
+       ['--show-conf','--show-config',
+        "[Pw] Show Pwrake configuration options",
+        lambda {|value| options.show_conf = true }
+       ]
+      ])
+      opts
+    end
+
+
+    def count(host_list, host)
+      @master.counter.count( host_list, host )
+    end
+
+    # from Michael Grosser's parallel
+    # https://github.com/grosser/parallel
+    def processor_count
+      host_os = RbConfig::CONFIG['host_os']
+      case host_os
+      when /linux|cygwin/
+        ncpu = 0
+        open("/proc/cpuinfo").each do |l|
+          ncpu += 1 if /^processor\s+: \d+/=~l
+        end
+        ncpu
+      when /darwin9/
+        `hwprefs cpu_count`.to_i
+      when /darwin/
+        (hwprefs_available? ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
+      when /(open|free)bsd/
+        `sysctl -n hw.ncpu`.to_i
+      when /mswin|mingw/
+        require 'win32ole'
+        wmi = WIN32OLE.connect("winmgmts://")
+        cpu = wmi.ExecQuery("select NumberOfLogicalProcessors from Win32_Processor")
+        cpu.to_enum.first.NumberOfLogicalProcessors
+      when /solaris2/
+        `psrinfo -p`.to_i # physical cpus
+      else
+        raise "Unknown architecture: #{host_os}"
+      end
+    end
+
+  end
+end

data/lib/pwrake/counter.rb

@@ -0,0 +1,54 @@
+module Pwrake
+
+  class Counter
+
+    def initialize
+      @same = 0
+      @diff = 0
+      @total = 0
+      @same_hosts = {}
+      @diff_hosts = {}
+      @no_queue = 0
+      @found_queue = 0
+      @empty_queue = 0
+    end
+
+    def count(host_list, host)
+      @total += 1
+      if host_list and host_list.include?(host)
+        @same += 1
+        @same_hosts[host] = (@same_hosts[host]||0) + 1
+      else
+        @diff += 1
+        @diff_hosts[host] = (@diff_hosts[host]||0) + 1
+      end
+    end
+
+    def print
+      s  = "same=#{@same}, diff=#{@diff}, total=#{@total}\n"
+      s << "same_hosts = {\n"
+      @same_hosts.keys.sort.each do |k|
+        s << "  #{k}: #{@same_hosts[k]}\n"
+      end
+      s << "}\n"
+      s << "different_hosts = {\n"
+      @diff_hosts.keys.sort.each do |k|
+        s << "  #{k}: #{@diff_hosts[k]}\n"
+      end
+      s << "}"
+      Log.info s
+    end
+
+    def no_queue
+      @no_queue += 1
+    end
+
+    def found_queue
+      @found_queue += 1
+    end
+
+    def empty_queue
+      @empty_queue += 1
+    end
+  end
+end

data/lib/pwrake/file_utils.rb

@@ -0,0 +1,70 @@
+module FileUtils
+
+  alias sh_orig :sh
+
+  def sh(*cmd, &block)
+    options = (Hash === cmd.last) ? cmd.pop : {}
+    unless block_given?
+      show_command = cmd.join(" ")
+      show_command = show_command[0,42] + "..."
+      block = lambda { |ok, status|
+        ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
+      }
+    end
+    if RakeFileUtils.verbose_flag == :default
+      options[:verbose] = true
+    else
+      options[:verbose] ||= RakeFileUtils.verbose_flag
+    end
+    options[:noop]    ||= RakeFileUtils.nowrite_flag
+    rake_check_options options, :noop, :verbose
+    Pwrake::Log.output_message cmd.join(" ") if options[:verbose]
+    unless options[:noop]
+      res,status = pwrake_system(*cmd)
+      block.call(res, status)
+    end
+  end
+
+  def pwrake_system(*cmd)
+    cmd_log = cmd.join(" ").inspect
+    tm = Pwrake::Timer.new("sh",cmd_log)
+
+    conn = Pwrake.current_shell
+    if conn.kind_of?(Pwrake::Shell)
+      res    = conn.system(*cmd)
+      status = Rake::PseudoStatus.new(conn.status)
+    else
+      res    = system(*cmd)
+      status = $?
+      status = Rake::PseudoStatus.new(1) if !res && status.nil?
+    end
+
+    tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
+    [res,status]
+  end
+  private :pwrake_system
+
+
+  # Pwrake version of backquote command
+  def pwrake_backquote(cmd)
+    cmd_log = cmd.inspect
+    tm = Pwrake::Timer.new("bq",cmd_log)
+
+    conn = Pwrake.current_shell
+    if conn.kind_of?(Pwrake::Shell)
+      res    = conn.backquote(*cmd)
+      status = conn.status
+    else
+      res    = `#{cmd}`
+      if !res && status.nil?
+        status = 1
+      else
+        status = $?.exitstatus
+      end
+    end
+
+    tm.finish("status=%s cmd=%s"%[status,cmd_log])
+    res
+  end
+
+end # module FileUtils

data/lib/pwrake/gfarm_feature.rb

@@ -0,0 +1,229 @@
+module Pwrake
+
+  module GfarmPath
+
+    module_function
+
+    def mountpoint_of_cwd
+      path = Pathname.pwd
+      while !path.mountpoint?
+        path = path.parent
+      end
+      path
+    end
+
+    @@local_mountpoint = mountpoint_of_cwd
+    @@fs_subdir = Pathname.new('/')
+
+    def mountpoint=(d)
+      @@local_mountpoint = Pathname.new(d)
+    end
+
+    def mountpoint
+      @@local_mountpoint
+    end
+
+    def subdir=(d)
+      if d
+        @@fs_subdir = Pathname.new(d)
+        if @@fs_subdir.relative?
+          @@fs_subdir = Pathname.new('/') + @@fs_subdir
+        end
+      end
+    end
+
+    def subdir
+      @@fs_subdir.to_s
+    end
+
+    def pwd
+      Pathname.pwd.relative_path_from(@@local_mountpoint)
+    end
+
+    def gfarm2fs?(d=nil)
+      d ||= @@local_mountpoint
+      mount_type = nil
+      open('/etc/mtab','r') do |f|
+        f.each_line do |l|
+          if /#{d} (?:type )?(\S+)/o =~ l
+            mount_type = $1
+            break
+          end
+        end
+      end
+      /gfarm2fs/ =~ mount_type
+    end
+
+    def from_local(x)
+      pn = Pathname(x)
+      if pn.absolute?
+        pn.relative_path_from(@@local_mountpoint)
+      else
+        Pathname.pwd.relative_path_from(@@local_mountpoint) + pn
+      end
+    end
+
+    def from_fs(x)
+      Pathname(x).relative_path_from(@@fs_subdir)
+    end
+
+    def to_fs(x)
+      @@fs_subdir + Pathname(x)
+    end
+
+    def to_local(x)
+      @@local_mountpoint + Pathname(x)
+    end
+
+    def local_to_fs(x)
+      x = from_local(x)
+      x = to_fs(x)
+      x.to_s
+    end
+
+    def fs_to_local(x)
+      x = from_fs(x)
+      x = to_local(x)
+      x.to_s
+    end
+
+    def gfwhere(list)
+      result = {}
+      count = 0
+      cmd = "gfwhere"
+      parse_proc = proc{|x|
+        if count==1
+          result[cmd[8..-1]] = x.split
+        else
+          x.scan(/^([^\n]+):\n([^\n]*)$/m) do |file,hosts|
+            h = hosts.split
+            result[file] = h if !h.empty?
+          end
+        end
+      }
+
+      list.each do |a|
+        if a
+          path = local_to_fs(a)
+          if cmd.size + path.size + 1 > 20480 # 131000
+            x = `#{cmd}`
+            parse_proc.call(x)
+            cmd = "gfwhere"
+            count = 0
+          end
+          cmd << " "
+          cmd << path
+          count += 1
+        end
+      end
+      if count > 0
+        x = `#{cmd}`
+        parse_proc.call(x)
+      end
+      result
+    end
+
+  end
+
+
+  class GfarmShell < Shell
+
+    @@core_id = {}
+    @@prefix = "pwrake_#{ENV['USER']}"
+
+    def initialize(host,opt={})
+      super(host,opt)
+      @single_mp = @option[:single_mp]
+      @basedir   = @option[:basedir]
+      @prefix    = @option[:prefix] || @@prefix
+      @work_dir  = @option[:work_dir]
+
+      @core_id = @@core_id[host] || 0
+      @@core_id[host] = @core_id + 1
+
+      if @single_mp
+        @remote_mountpoint = "#{@basedir}/#{@prefix}_00"
+      else
+        @remote_mountpoint = "#{@basedir}/#{@prefix}_%02d" % @core_id
+      end
+    end
+
+    def start
+      Log.debug "--- mountpoint=#{@remote_mountpoint}"
+      open(system_cmd)
+      cd
+      if not _system "test -d #{@remote_mountpoint}"
+        _system "mkdir -p #{@remote_mountpoint}"
+        subdir = GfarmPath.subdir
+        if ["/","",nil].include?(subdir)
+          _system "gfarm2fs #{@remote_mountpoint}"
+        else
+          _system "gfarm2fs -o modules=subdir,subdir=#{subdir} #{@remote_mountpoint}"
+        end
+      end
+      path = ENV['PATH'].gsub( /#{GfarmPath.mountpoint}/, @remote_mountpoint )
+      _system "export PATH=#{path}"
+      cd_work_dir
+    end
+
+    def close
+      if @remote_mountpoint
+        cd
+        _system "fusermount -u #{@remote_mountpoint}"
+        _system "rmdir #{@remote_mountpoint}"
+      end
+      super
+      self
+    end
+
+    def cd_work_dir
+      # modify local work_dir -> remote work_dir
+      dir = Pathname.new(@remote_mountpoint) + GfarmPath.pwd
+      cd dir
+    end
+
+  end
+
+
+  class GfarmQueue < LocalityAwareQueue
+
+    def abr_msg(a)
+      m = a[0..5].map{|x| x}.inspect
+      m.sub!(/]$/,",...") if a.size > 6
+      "size=#{a.size} #{m}"
+    end
+
+    def where(tasks)
+      if Pwrake.application.options.dryrun ||
+          Pwrake.application.options.disable_affinity
+        return tasks
+      end
+
+      start_time = Time.now
+      #Log.debug "--- GfarmQueue#where #{tasks.inspect}"
+      #if Pwrake.manager.gfarm and Pwrake.manager.affinity
+      gfwhere_result = {}
+      filenames = []
+      tasks.each do |t|
+        if t.kind_of?(Rake::FileTask) and
+            name = t.prerequisites[0] and
+            !filenames.include?(name)
+          filenames << name
+        end
+      end
+
+      if !filenames.empty?
+        gfwhere_result = GfarmPath.gfwhere(filenames)
+        tasks.each do |t|
+          if t.kind_of? Rake::FileTask and prereq_name = t.prerequisites[0]
+            t.location = gfwhere_result[GfarmPath.local_to_fs(prereq_name)]
+          end
+        end
+      end
+      Log.info "-- GfarmQueue#where %.6fs %s" % [Time.now-start_time,abr_msg(filenames)]
+      tasks
+    end
+
+  end
+
+end