pwrake 2.0.1 → 2.1.0

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.0.1"
+  VERSION = "2.1.0"
 end

data/lib/pwrake/worker/invoker.rb

@@ -1,6 +1,9 @@
 module Pwrake
 
   class Invoker
+    # using Michael Grosser's parallel
+    # https://github.com/grosser/parallel
+    include Parallel::ProcessorCount
 
     def initialize(dir_class, ncore, option)
       @dir_class = dir_class
@@ -161,37 +164,9 @@ module Pwrake
         $stdout.puts e
         $stdout.puts e.backtrace.join("\n")
       end
+    ensure
       @out.puts "exited"
     end
 
-    # from Michael Grosser's parallel
-    # https://github.com/grosser/parallel
-    def processor_count
-      host_os = RbConfig::CONFIG['host_os']
-      case host_os
-      when /linux|cygwin/
-        ncpu = 0
-        open("/proc/cpuinfo").each do |l|
-          ncpu += 1 if /^processor\s+: \d+/=~l
-        end
-        ncpu
-      when /darwin9/
-        `hwprefs cpu_count`.to_i
-      when /darwin/
-        (hwprefs_available? ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
-      when /(open|free)bsd/
-        `sysctl -n hw.ncpu`.to_i
-      when /mswin|mingw/
-        require 'win32ole'
-        wmi = WIN32OLE.connect("winmgmts://")
-        cpu = wmi.ExecQuery("select NumberOfLogicalProcessors from Win32_Processor")
-        cpu.to_enum.first.NumberOfLogicalProcessors
-      when /solaris2/
-        `psrinfo -p`.to_i # physical cpus
-      else
-        raise "Unknown architecture: #{host_os}"
-      end
-    end
-
   end
 end

data/pwrake.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |gem|
   gem.version       = Pwrake::VERSION
   gem.authors       = ["Masahiro TANAKA"]
   gem.email         = ["masa16.tanaka@gmail.com"]
-  gem.description   = %q{Parallel workflow extension for Rake}
-  gem.summary       = %q{Adding Parallel and Distributed feature to Rake}
+  gem.summary       = %q{Parallel Workflow engine based on Rake}
+  gem.description   = %q{Parallel Workflow engine based on Rake, runs on multicores, clusters, clouds}
   gem.homepage      = "http://masa16.github.com/pwrake"
   gem.license       = 'MIT'
 
@@ -17,4 +17,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+  gem.add_runtime_dependency 'parallel', '>= 1.2.4'
 end

metadata

@@ -1,16 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-02 00:00:00.000000000 Z
-dependencies: []
-description: Parallel workflow extension for Rake
+date: 2016-05-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+description: Parallel Workflow engine based on Rake, runs on multicores, clusters,
+  clouds
 email:
 - masa16.tanaka@gmail.com
 executables:
@@ -29,22 +44,17 @@ files:
 - bin/gfwhere-pipe
 - bin/pwrake
 - bin/pwrake_branch
-- lib/pwrake/branch.rb
 - lib/pwrake/branch/branch.rb
 - lib/pwrake/branch/branch_application.rb
+- lib/pwrake/branch/communicator.rb
+- lib/pwrake/branch/communicator_set.rb
 - lib/pwrake/branch/fiber_queue.rb
 - lib/pwrake/branch/file_utils.rb
 - lib/pwrake/branch/shell.rb
 - lib/pwrake/branch/shell_profiler.rb
-- lib/pwrake/branch/worker_communicator.rb
 - lib/pwrake/gfarm/gfarm_path.rb
 - lib/pwrake/gfarm/gfarm_postprocess.rb
-- lib/pwrake/iomux/channel.rb
-- lib/pwrake/iomux/handler.rb
-- lib/pwrake/iomux/handler_set.rb
-- lib/pwrake/iomux/runner.rb
 - lib/pwrake/logger.rb
-- lib/pwrake/master.rb
 - lib/pwrake/master/fiber_pool.rb
 - lib/pwrake/master/idle_cores.rb
 - lib/pwrake/master/master.rb
@@ -52,6 +62,7 @@ files:
 - lib/pwrake/master/postprocess.rb
 - lib/pwrake/misc/graphviz.rb
 - lib/pwrake/misc/mcgp.rb
+- lib/pwrake/nbio.rb
 - lib/pwrake/option/host_map.rb
 - lib/pwrake/option/option.rb
 - lib/pwrake/option/option_filesystem.rb
@@ -121,7 +132,7 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Adding Parallel and Distributed feature to Rake
+summary: Parallel Workflow engine based on Rake
 test_files:
 - spec/001/Rakefile
 - spec/002/Rakefile
@@ -140,3 +151,4 @@ test_files:
 - spec/helper.rb
 - spec/hosts
 - spec/pwrake_spec.rb
+has_rdoc: 

data/lib/pwrake/branch.rb

@@ -1,22 +0,0 @@
-require "logger"
-require 'csv'
-
-require "pwrake/logger"
-
-require "pwrake/iomux/channel"
-require "pwrake/iomux/handler"
-require "pwrake/iomux/handler_set"
-require "pwrake/iomux/runner"
-
-require "pwrake/branch/branch_application"
-require "pwrake/branch/branch"
-require "pwrake/branch/fiber_queue"
-require "pwrake/branch/file_utils"
-
-require 'pwrake/branch/shell_profiler'
-require "pwrake/branch/shell"
-require "pwrake/branch/worker_communicator"
-
-require "pwrake/option/option"
-require "pwrake/option/option_filesystem"
-require "pwrake/option/host_map"

data/lib/pwrake/branch/worker_communicator.rb

@@ -1,104 +0,0 @@
-module Pwrake
-
-  class WorkerCommunicator
-
-    HOST2ID = {}
-    RE_ID='\d+'
-    attr_reader :id, :host, :ncore, :handler, :channel
-
-    def self.read_worker_progs(option)
-      d = File.dirname(__FILE__)+'/../worker/'
-      code = ""
-      option.worker_progs.each do |f|
-        code << IO.read(d+f+'.rb')
-      end
-      code
-    end
-
-    def initialize(id,host,ncore,runner,option)
-      @id = id
-      @host = host
-      HOST2ID[@host] = @id
-      @ncore = @n_total_core = ncore
-      #
-      @runner = runner
-      @worker_progs = option.worker_progs
-      @option = option.worker_option
-      if hb = @option[:heartbeat]
-        @heartbeat_timeout = hb + 15
-      end
-    end
-
-    def setup_connection(worker_code)
-      rb_cmd = "ruby -e 'eval ARGF.read(#{worker_code.size})'"
-      if ['localhost','localhost.localdomain','127.0.0.1'].include? @host
-        cmd = "cd; #{rb_cmd}"
-      else
-        cmd = "ssh -x -T -q #{@option[:ssh_option]} #{@host} \"#{rb_cmd}\""
-      end
-      Log.debug cmd
-      @handler = Handler.new(@runner) do |w0,w1,r2|
-        @pid = spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
-        w0.close
-        w1.close
-        r2.close
-      end
-      @handler.host = @host
-      iow = @handler.iow
-      iow.write(worker_code)
-      Marshal.dump(@ncore,iow)
-      Marshal.dump(@option,iow)
-      @channel = Channel.new(@handler)
-    end
-
-    def close
-      if !@closed
-        @closed = true
-        @handler.close
-      end
-    end
-
-    def set_ncore(ncore)
-      @ncore = ncore
-    end
-
-    def ncore_proc(s)
-      if /^ncore:(.*)$/ =~ s
-        a = $1
-        Log.debug "#{a} @#{@host}"
-        if /^(\d+)$/ =~ a
-          set_ncore($1.to_i)
-          return false
-        else
-          raise ArgumentError,"WorkerCommunicator#ncore_proc: s=#{s.inspect}"
-        end
-      else
-        return common_line(s)
-      end
-    end
-
-    def common_line(s)
-      Log.debug "WorkerCommunicator#common_line(#{s.inspect}) id=#{@id} host=#{@host}"
-      case s
-      when /^heartbeat$/
-        @runner.heartbeat(@handler.ior)
-      when /^exited$/
-        return false
-      when /^log:(.*)$/
-        Log.info "worker(#{host})>#{$1}"
-      else
-        Log.warn "worker(#{host}) out> #{s.inspect}"
-      end
-      true
-    end
-
-    def start_default_fiber
-      Fiber.new do
-        while common_line(@channel.get_line)
-        end
-        Log.debug "WorkerCommunicator: end of default fiber"
-      end.resume
-    end
-
-  end
-end

data/lib/pwrake/iomux/channel.rb

@@ -1,70 +0,0 @@
-module Pwrake
-
-  class Channel
-
-    def initialize(handler,id=nil)
-      if @id = id
-        @pre = "#{@id}:"
-      else
-        @pre = ""
-      end
-      if !handler.kind_of?(Handler)
-        raise TypeError, "Argument must be Handler but #{handler.class}"
-      end
-      @handler = handler
-    end
-
-    attr_reader :id, :handler, :fiber
-
-    def ior
-      @handler.ior
-    end
-
-    def run_fiber(*args)
-      if @fiber.nil?
-        m = "Channel#run_fiber: @fiber is nil,"+
-          " args=#{args.inspect} @id=#{@id}"
-        $stderr.puts m
-        Log.debug m
-      else
-        @fiber.resume(*args)
-      end
-    end
-
-    def finish
-      if !@fiber.nil?
-        @fiber.resume(nil)
-      end
-    end
-
-    def get_line
-      @handler.add_channel(self)
-      @fiber = Fiber.current
-      line = Fiber.yield
-      @fiber = nil
-      @handler.delete_channel(self)
-      return line
-    end
-
-    def put_line(line)
-      @handler.put_line "#{@pre}#{line}"
-    end
-
-    def puts(line)
-      @handler.puts "#{@pre}#{line}"
-    end
-
-    def gets
-      if @id
-        raise RuntimeError,"gets is invalid when @id is non-nil"
-      else
-        @handler.gets
-      end
-    end
-
-    def inspect
-      "#<#{self.class} io=#{ior.inspect} id=#{id.inspect}>"
-    end
-  end
-
-end

data/lib/pwrake/iomux/handler.rb

@@ -1,124 +0,0 @@
-module Pwrake
-
-  class Handler
-
-    def initialize(runner,*io,&blk)
-      if !runner.kind_of?(Runner)
-        raise TypeError, "Argument must be Runner but #{runner.class}"
-      end
-      @runner = runner
-      @channel = {}
-      if !io.empty?
-        if !block_given?
-          @ior,@iow,@ioe = *io
-        end
-      else
-        if block_given?
-          @ior,w0 = IO.pipe
-          @ioe,w1 = IO.pipe
-          r2,@iow = IO.pipe
-          yield(w0,w1,r2)
-        end
-      end
-      if @ior.nil? && @iow.nil?
-        raise "fail to initialize IO"
-      end
-    end
-
-    attr_reader :runner, :ior, :iow, :ioe
-    attr_accessor :host
-
-    def add_channel(chan)
-      if !chan.kind_of?(Channel)
-        raise TypeError, "Argument must be Channel but #{chan.class}"
-      end
-      @channel[chan.id] = chan
-      @runner.add_handler(self)
-    end
-
-    def delete_channel(chan)
-      if !chan.kind_of?(Channel)
-        raise TypeError, "Argument must be Channel but #{chan.class}"
-      end
-      @channel.delete(chan.id)
-      if @channel.empty?
-        @runner.delete_handler(self)
-      end
-    end
-
-    def process_line
-      if s = (@ior.eof?) ? nil : @ior.gets
-        default_channel = @channel[nil]
-        if @channel.size > ((default_channel) ? 1 : 0)
-          if /^(\d+):(.*)$/ =~ s
-            ch,line = $1,$2
-            if chan = @channel[ch.to_i]
-              return chan.run_fiber(line)
-            else
-              raise "No channel##{ch}"
-            end
-          end
-        end
-        if default_channel
-          return default_channel.run_fiber(s.chomp)
-        else
-          raise "No default_channel, received: #{s.chomp}"
-        end
-      else
-        # End of IO
-        @channel.values.each do |chan|
-          if chan.fiber
-            chan.run_fiber(nil)
-          end
-        end
-      end
-    end
-
-    def finish
-      @channel.values.each do |chan|
-        chan.finish
-      end
-    end
-
-# -- writer
-
-    def put_line(line)
-      begin
-        @iow.print line.to_str+"\n"
-        @iow.flush
-        #Log.debug "Handler#put_line #{line.inspect} @iow=#{@iow.inspect}"
-      rescue Errno::EPIPE => e
-        if Rake.application.options.debug
-          $stderr.puts "Errno::EPIPE in #{self.class}.put_line '#{line.chomp}'"
-          $stderr.puts e.backtrace.join("\n")
-        end
-        Log.error "Errno::EPIPE in #{self.class}.put_line '#{line.chomp}'\n"+
-          e.backtrace.join("\n")
-        #raise e
-      end
-    end
-
-    def puts(line)
-      @iow.puts(line)
-    end
-
-    def gets
-      @ior.gets
-    end
-
-    def wait_message(end_msg)
-      if line = @ior.gets
-        line.chomp!
-        m = "Handler#wait_message: #{line} host=#{@host}"
-        if line == end_msg
-          Log.debug m
-        else
-          Log.error m
-        end
-      else
-        Log.error "Handler#wait_message: fail to read @ior"
-      end
-    end
-
-  end
-end