pwrake 2.2.6 → 2.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2c9f51a1a5bdbb546c288239917a1284a7d831f4332c339474d5eb4e368a0ac
-  data.tar.gz: dda35b31e72bd380b0e6ee5e1e2d6cdecb66698bfd3167ce792e3b4f95146f4c
+  metadata.gz: 20590e6007af5bc0c567fe8266a82f40aaeec0bbb853bdd87e180f0f1c53bc31
+  data.tar.gz: 4ec4760f2428d854019675a2f51e49f791102f7eab6b3141e258cfbc25a8842a
 SHA512:
-  metadata.gz: 5999d1c3ac6217ebab9d5e1915f6f9cf6c8c2023fa3fe973a3a3de7dfe751f4b89e5c8b1a31d4a0a9e972a8fb1be8233fb2c357b189f339925194bfde2352e52
-  data.tar.gz: 366a8e85bf26d3af24b9f80f4521c8b222c56985b50c644f337d4e996eb232f8c077df6fbe4a314b5794eb91d840639ce8c13a3e508fc695ea12bf3c4cbd3fad
+  metadata.gz: 391db5799b9091fbbc0a242f78729fe265087581b6b88dbaf9bf01f188e8d4412c749683075c38dfcea4803589289205f85c42320913727fe51628eb7fdaa2f8
+  data.tar.gz: fc7a41165bdce7fa685e3d922d5b24f6146524deaa52a31049c459d8fa39430b3948b4d02943ec143014181f2f916e3ab6d06f9596b61c05d397243a02770fe4

data/README.md

@@ -23,6 +23,14 @@ Parallel Workflow extension for Rake, runs on multicores, clusters, clouds.
   * Pwrake schedules a compute node to execute a task, to a node where input files are stored.
   * Other supports for Gfarm: Automatic mount of the Gfarm file system, etc.
 
+## Requirement
+
+* Ruby version 2.2.3 or later
+* UNIX-like OS
+* For distributed processing using multiple computers:
+  * SSH command
+  * distributed file system (NFS, Gfarm, etc.)
+
 ## Installation
 
 Install with RubyGems:
@@ -33,8 +41,6 @@ Or download source tgz/zip and expand, cd to subdirectory and install:
 
     $ ruby setup.rb
 
-In the latter case, you need install [Parallel](https://github.com/grosser/parallel) manually. It is required by Pwrake for processor count.
-
 If you use rbenv, your system may fail to find pwrake command after installation:
 
     -bash: pwrake: command not found
@@ -70,7 +76,7 @@ In this case, you need the rehash of command paths:
 
         $ pwrake -F hosts
 
-### Use MPI to start remote worker
+### Sustitute MPI for SSH to start remote worker (Experimental)
 
 1. Setup MPI on your cluster.
 2. Install [MPipe gem](https://rubygems.org/gems/mpipe). (requires `mpicc`)
@@ -179,10 +185,9 @@ Properties (The leftmost item is default):
 
 ## Note for Gfarm
 
-* `gfwhere-pipe` script (included in Pwrake) is used for file-affinity scheduling.
-  This script requires Ruby/FFI (https://github.com/ffi/ffi). Install FFI by
-
-        gem install ffi
+* Gfarm file-affinity scheduling is achieved by `gfwhere-pipe` script bundled in the Pwrake package.
+  This script accesses `libgfarm.so.1` through Fiddle (a Ruby's standard module) since Pwrake ver.2.2.7.
+  Please set the environment variable `LD_LIBRARY_PATH` correctly to find `libgfarm.so.1`.
 
 ## Scheduling with Graph Partitioning
 
@@ -200,16 +205,7 @@ Properties (The leftmost item is default):
 
 * See publication: [M. Tanaka and O. Tatebe, “Workflow Scheduling to Minimize Data Movement Using Multi-constraint Graph Partitioning,” in CCGrid 2012](http://ieeexplore.ieee.org/abstract/document/6217406/)
 
-## Current version
-
-* Pwrake version 2.2.3
-
-## Tested Platform
-
-
-* Ruby 2.4.1
-* Rake 12.0.0
-* CentOS 7.3
+## [Publications](https://github.com/masa16/pwrake/wiki/Publications)
 
 ## Acknowledgment
 

data/bin/gfwhere-pipe

@@ -1,15 +1,18 @@
 #! /usr/bin/env ruby
 
-require 'ffi'
-require 'singleton'
+require 'fiddle'
 
 module Gfarm
 
   class GfarmError < StandardError
   end
+
   GFARM_ERR_NO_ERROR = 0
 
-  module FFI
+  module LibGfarm
+
+    module_function
+
     def find_executable(name)
       path = "/usr/local/bin:/usr/ucb:/usr/bin:/bin"
       begin
@@ -24,171 +27,180 @@ module Gfarm
         begin
           stat = File.stat(file)
         rescue SystemCallError
-      else
+        else
           return file if stat.file? and stat.executable?
         end
       end
       nil
     end
-    module_function :find_executable
-
-    if LIBGFARM_PATH = ENV['LIBGFARM_PATH']
-      dirs = LIBGFARM_PATH.split(":")
-    elsif d = find_executable('gfwhere')
-      d = File.dirname(File.dirname(d))
-      dirs = %w[lib64 lib].map{|l| File.join(d,l)}
-    else
-      raise StandardError, "cannot find libgfarm path"
-    end
-    path = nil
-    dirs.each do |d|
-      f = File.join(d,"libgfarm.so*")
-      g = Dir.glob(f)
-      if !g.empty?
-        path = g[0]
-        break
+
+    begin
+      HANDLE = Fiddle::Handle.new('libgfarm.so.1')
+    rescue
+      if d = find_executable('gfwhere')
+        d = File.dirname(File.dirname(d))
+        dirs = %w[lib64 lib].map{|l| File.join(d,l)}
+      else
+        raise StandardError, "cannot find libgfarm path"
+      end
+      path = nil
+      dirs.each do |d|
+        f = File.join(d,"libgfarm.so*")
+        g = Dir.glob(f)
+        if !g.empty?
+          path = g[0]
+          break
+        end
+      end
+      if !(path && File.exist?(path))
+        raise StandardError, "cannot find libgfarm"
       end
+      HANDLE = Fiddle::Handle.new(path)
     end
-    if !(path && File.exist?(path))
-      raise StandardError, "cannot find libgfarm"
+
+    FUNC = {}
+
+    def def_cfunc(name,argtypes,rettype=Fiddle::TYPE_INT)
+      FUNC[name] = func =
+        Fiddle::Function.new(HANDLE.sym(name),argtypes,rettype,name:name)
+      define_singleton_method(name){|*a| func.call(*a)}
     end
-    extend ::FFI::Library
-    ffi_lib path
-    attach_function :gfarm_initialize, [:pointer, :pointer], :int
-    attach_function :gfarm_terminate, [], :int
-    attach_function :gfarm_realpath_by_gfarm2fs, [:string, :pointer], :int
-    attach_function :gfarm_error_string, [:int], :string
-    attach_function :gfs_replica_info_by_name, [:string, :int, :pointer], :int
-    attach_function :gfs_replica_info_number, [:pointer], :int
-    attach_function :gfs_replica_info_free, [:pointer], :void
-    attach_function :gfs_replica_info_nth_host, [:pointer, :int], :string
-  end
 
+    # gfarm_error_t gfarm_initialize(int *argcp, char *** argvp);
+    def_cfunc 'gfarm_initialize',[Fiddle::TYPE_VOIDP,Fiddle::TYPE_VOIDP]
 
-  class Connection
-    include Singleton
+    # gfarm_error_t gfarm_terminate(void);
+    def_cfunc 'gfarm_terminate',[]
 
-    def self.callback
-      proc{ FFI.gfarm_terminate }
-    end
+    # const char *gfarm_error_string(gfarm_error_t);
+    def_cfunc 'gfarm_error_string',[Fiddle::TYPE_INT],Fiddle::TYPE_VOIDP
+
+    # gfarm_error_t gfarm_realpath_by_gfarm2fs(const char *, char **);
+    def_cfunc 'gfarm_realpath_by_gfarm2fs',[Fiddle::TYPE_VOIDP,Fiddle::TYPE_VOIDP]
+
+    # gfarm_error_t gfs_replica_info_by_name(
+    #   const char *, int, struct gfs_replica_info **);
+    def_cfunc 'gfs_replica_info_by_name',[Fiddle::TYPE_VOIDP,Fiddle::TYPE_INT,Fiddle::TYPE_VOIDP]
+
+    # void gfs_replica_info_free(struct gfs_replica_info *);
+    def_cfunc 'gfs_replica_info_free',[Fiddle::TYPE_VOIDP],Fiddle::TYPE_VOID
+
+    # int gfs_replica_info_number(struct gfs_replica_info *);
+    def_cfunc 'gfs_replica_info_number',[Fiddle::TYPE_VOIDP]
+
+    # const char *gfs_replica_info_nth_host(struct gfs_replica_info *, int);
+    def_cfunc 'gfs_replica_info_nth_host',[Fiddle::TYPE_VOIDP,Fiddle::TYPE_INT],Fiddle::TYPE_VOIDP
 
-    def self.set_args(args)
-      @@args = args
-    end
 
-    def initialize
-      args = @@args || []
-      argc = ::FFI::MemoryPointer.new(:int, 1)
-      argc.write_int(args.size)
-      ary = args.map do |s|
-        str = ::FFI::MemoryPointer.new(:string, s.size)
-        str.write_string(s)
-        str
+    @@initialized = false
+
+    def initialize(*argv)
+      if @@initialized
+        warn "gfarm_initialize: already initialized"
+        return
+      end
+      argc_buf = [argv.size].pack('i')
+      if argv.empty?
+        argv_ary = [0].pack('J')
+      else
+        argv_ary = ARGV.map{|a| Fiddle::Pointer[a]}.pack('J*')
       end
-      ptr = ::FFI::MemoryPointer.new(:pointer, args.size)
-      ptr.write_array_of_pointer(ary)
-      argv = ::FFI::MemoryPointer.new(:pointer, 1)
-      argv.write_pointer(ptr)
-      e = FFI.gfarm_initialize(argc, argv)
+      argv_buf = [Fiddle::Pointer[argv_ary]].pack('J')
+      e = LibGfarm.gfarm_initialize(argc_buf,argv_buf)
+      # size = argc_buf.unpack('i').first
       if e != GFARM_ERR_NO_ERROR
-        raise GfarmError, FFI.gfarm_error_string(e)
+        raise GfarmError,error_string(e)
       end
-      ObjectSpace.define_finalizer(self, self.class.callback)
+      @@initialized = true
+      at_exit{ gfarm_terminate() }
+    end
+
+    def error_string(i)
+      gfarm_error_string(i).to_s
     end
 
     def realpath_by_gfarm2fs(path)
-      ptr = ::FFI::MemoryPointer.new(:pointer, 1)
-      e = FFI.gfarm_realpath_by_gfarm2fs(path, ptr)
+      ptr_buf = [0].pack('J')
+      e = gfarm_realpath_by_gfarm2fs(path,ptr_buf)
       if e != GFARM_ERR_NO_ERROR
-        raise GfarmError, FFI.gfarm_error_string(e)
+        raise GfarmError,error_string(e)
       end
-      ptr.read_pointer().read_string()
+      Fiddle::Pointer[ptr_buf.unpack('J').first].to_s
     end
 
-    def replica_info_by_name(name)
-      ReplicaInfo.new(self,name)
-    end
   end
 
 
-  class ReplicaInfo < ::FFI::AutoPointer
+  class ReplicaInfo
 
-    def self.release(ptr)
-      FFI.gfs_replica_info_free(ptr)
-    end
+    @@flags = 0
+    INCLUDING_DEAD_HOST       = 1
+    INCLUDING_INCOMPLETE_COPY = 2
+    INCLUDING_DEAD_COPY       = 4
 
-    def self.set_opts(opts)
-      @@opts = opts
+    def self.set_opts(argv)
+      @@flags = 0
+      args = []
+      argv.each do |x|
+        case x
+        when "-i"
+          @@flags |= INCLUDING_INCOMPLETE_COPY
+        else
+          args << x
+        end
+      end
+      args
     end
 
-    def initialize(gfarm, path)
-      @gfarm = gfarm
-      @realpath = @gfarm.realpath_by_gfarm2fs(path)
-      flag = @@opts.flags
-      ptr = ::FFI::MemoryPointer.new(:pointer, 1)
-      e = FFI.gfs_replica_info_by_name(@realpath, flag, ptr)
+    def initialize(path)
+      while File.symlink?(path)
+        link = File.readlink(path)
+        path = File.expand_path(link, File.dirname(path))
+      end
+      @realpath = LibGfarm.realpath_by_gfarm2fs(path)
+      ptr_buf = [0].pack('J')
+      e = LibGfarm.gfs_replica_info_by_name(@realpath,@@flags,ptr_buf)
       if e != GFARM_ERR_NO_ERROR
-        raise GfarmError, @realpath+" "+FFI.gfarm_error_string(e)
+        raise GfarmError,@realpath+" : "+LibGfarm.error_string(e)
       end
-      @ri = ptr.read_pointer()
-      super @ri
+      @ptr = Fiddle::Pointer[ptr_buf.unpack('J').first]
+      @ptr.free = LibGfarm::FUNC['gfs_replica_info_free']
     end
+
     attr_reader :realpath
 
     def number
-      FFI.gfs_replica_info_number(@ri)
+      LibGfarm.gfs_replica_info_number(@ptr)
     end
 
     def nth_host(i)
-      FFI.gfs_replica_info_nth_host(@ri,i)
+      LibGfarm.gfs_replica_info_nth_host(@ptr,i).to_s
     end
   end
 
-  class Options
-    INCLUDING_DEAD_HOST       = 1
-    INCLUDING_INCOMPLETE_COPY = 2
-    INCLUDING_DEAD_COPY       = 4
+end
 
-    def initialize(argv)
-      @args = []
-      @flags = 0
-      argv.each do |x|
-        case x
-        when "-i"
-          @including_incomplete_copy = true
-          @flags |= INCLUDING_INCOMPLETE_COPY
-        else
-          @args << x
-        end
-      end
-    end
+if $0 == __FILE__
 
-    attr_reader :args
-    attr_reader :flags
-    attr_reader :including_incomplete_copy
+  [:PIPE,:TERM,:INT].each do |sig|
+    Signal.trap(sig, "EXIT")
   end
 
-end
-
-[:PIPE,:TERM,:INT].each do |sig|
-  Signal.trap(sig, "EXIT")
-end
-
-opts = Gfarm::Options.new(ARGV)
-Gfarm::ReplicaInfo.set_opts(opts)
-Gfarm::Connection.set_args(opts.args)
-gfarm = Gfarm::Connection.instance
-
-while path=$stdin.gets
-  path.chomp!
-  $stdout.print path+"\n"
-  $stdout.flush
-  begin
-    ri = gfarm.replica_info_by_name(path)
-    hosts = ri.number.times.map{|i| ri.nth_host(i) }
-    $stdout.print ri.realpath+":\n"+hosts.join(" ")+"\n"
-  rescue
-    $stdout.print "Error: "+path+"\n"
+  argv = Gfarm::ReplicaInfo.set_opts(ARGV)
+  Gfarm::LibGfarm.initialize(*argv)
+
+  while path=$stdin.gets
+    path.chomp!
+    $stdout.print path+"\n"
+    $stdout.flush
+    begin
+      rep_info = Gfarm::ReplicaInfo.new(path)
+      hosts = rep_info.number.times.map{|i| rep_info.nth_host(i) }
+      $stdout.print rep_info.realpath+":\n"+hosts.join(" ")+"\n"
+    rescue
+      $stdout.print "Error: "+path+"\n"
+    end
+    $stdout.flush
   end
-  $stdout.flush
+
 end

data/lib/pwrake/master/master_application.rb

@@ -22,7 +22,6 @@ module Pwrake
       standard_exception_handling do
         init("pwrake")  # <- parse options here
         @role = @master = Master.new
-        t = Time.now
         t = Pwrake.clock
         @master.setup_branches
         load_rakefile

data/lib/pwrake/mpi/worker.rb

@@ -1,4 +1,3 @@
-require "parallel/processor_count.rb"
 require "pwrake/nbio"
 require "pwrake/branch/fiber_queue"
 require "pwrake/worker/writer"

data/lib/pwrake/option/option.rb

@@ -1,7 +1,6 @@
 require "pathname"
 require "yaml"
 require "socket"
-require "parallel"
 require "pwrake/option/host_map"
 
 module Pwrake

data/lib/pwrake/option/option_default_filesystem.rb

@@ -10,7 +10,6 @@ module Pwrake
 
     def set_filesystem_option
       @worker_progs = %w[
-        parallel/processor_count.rb
         pwrake/nbio
         pwrake/branch/fiber_queue
         pwrake/worker/writer

data/lib/pwrake/option/option_gfarm2fs.rb

@@ -39,7 +39,6 @@ module Pwrake
         :single_mp => self['GFARM_SINGLE_MP']
       }
       @worker_progs = %w[
-        parallel/processor_count.rb
         pwrake/nbio
         pwrake/branch/fiber_queue
         pwrake/worker/writer

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.2.6"
+  VERSION = "2.2.7"
 end

data/lib/pwrake/worker/invoker.rb

@@ -1,14 +1,13 @@
 require "socket"
+require "etc"
 
 module Pwrake
 
   class Invoker
-    begin
-      # use Michael Grosser's Parallel module
-      # https://github.com/grosser/parallel
-      include Parallel::ProcessorCount
-    rescue
-      def processor_count
+    def processor_count
+      begin
+        Etc.nprocessors
+      rescue
         # only for Linux
         IO.read("/proc/cpuinfo").scan(/^processor/).size
       end

data/pwrake.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |gem|
   gem.version       = Pwrake::VERSION
   gem.authors       = ["Masahiro TANAKA"]
   gem.email         = ["masa16.tanaka@gmail.com"]
-  gem.summary       = %q{Parallel Workflow engine based on Rake}
-  gem.description   = %q{Parallel Workflow engine based on Rake, runs on multicores, clusters, clouds}
+  gem.summary       = %q{Parallel and distributed Rake for workflow execution on multicores, clusters, clouds.}
+  gem.description   = %q{Parallel and distributed Rake for workflow execution on multicores, clusters, clouds using SSH. It has locality-aware scheduling designed for Gfarm file system.}
   gem.homepage      = "http://masa16.github.com/pwrake"
   gem.license       = 'MIT'
 
@@ -17,5 +17,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency 'parallel', '>= 1.2.4'
+  gem.required_ruby_version = '>= 2.2.3'
 end

metadata

@@ -1,31 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.2.6
+  version: 2.2.7
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-15 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.4
-description: Parallel Workflow engine based on Rake, runs on multicores, clusters,
-  clouds
+date: 2018-11-30 00:00:00.000000000 Z
+dependencies: []
+description: Parallel and distributed Rake for workflow execution on multicores, clusters,
+  clouds using SSH. It has locality-aware scheduling designed for Gfarm file system.
 email:
 - masa16.tanaka@gmail.com
 executables:
@@ -128,7 +114,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.2.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -136,10 +122,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: Parallel Workflow engine based on Rake
+summary: Parallel and distributed Rake for workflow execution on multicores, clusters,
+  clouds.
 test_files:
 - spec/001/Rakefile
 - spec/002/Rakefile