pwrake 0.9.9.2 → 2.0.0

data/spec/003/Rakefile

@@ -1,9 +1,9 @@
 task :first do |t, args|
-  puts "#{args.x},#{args.y}"
+  puts "first x=#{args.x},y=#{args.y}"
 end
 
 task :hello, ['x', 'y'] => :first do |t, args|
-  puts "#{args.x},#{args.y}"
+  puts "hello x=#{args.x},y=#{args.y}"
 end
 
 task :default => :hello

data/spec/008/Rakefile

@@ -1,3 +1,4 @@
 task :default do
-  puts "this message should not be printed"
+  #puts "this message should not be printed"
+  sh "hostname; echo ENV1=$ENV1"
 end

data/spec/009/Rakefile

@@ -15,4 +15,4 @@ end
 task :default => A
 
 CLEAN.include A,B
-CLEAN.include '*.csv'
+CLEAN.include 'Pwrake*'

data/spec/009/pwrake_conf.yaml

@@ -1,3 +1 @@
-PROFILE : true
-GNU_TIME : true
-#TASKLOG : true
+LOG_DIR: true

data/spec/hosts

@@ -1,3 +1 @@
 localhost 4
-#tsukuba001 2
-#tsukuba002 2

data/spec/pwrake_spec.rb

@@ -1,9 +1,10 @@
+require "rspec/its"
+
 $spec_dir = File.expand_path(File.dirname(__FILE__))
 require $spec_dir+"/helper"
 $hosts = $spec_dir+"/hosts"
 
 RSpec.configure do |config|
-  config.treat_symbols_as_metadata_keys_with_true_values = true
   config.filter_run :focus
   config.run_all_when_everything_filtered = true
 end
@@ -38,7 +39,7 @@ describe Helper do
   context "dir=003 w task argument" do
     subject { Helper.new("003","hello[foo,bar]").run }
     it { should be_success }
-    its(:result) { should eq "foo,bar\nfoo,bar\n" }
+    its(:result) { should eq "first x=foo,y=bar\nhello x=foo,y=bar\n" }
   end
 
   context "dir=004 -j4 elapsed time" do
@@ -61,12 +62,12 @@ describe Helper do
     its(:result) { should eq "pass_successfully\n" }
   end
 
-  # context "dir=007 invoke-in-task", :focus=>true do
-  context "dir=007 invoke-in-task" do
-    subject { Helper.new("007","-j10").run }
-    it { should be_success }
-    its(:elapsed_time) { should be_within(1).of(2) } # 1..3 sec
-  end
+  ## context "dir=007 invoke-in-task", :focus=>true do
+  #context "dir=007 invoke-in-task" do
+  #  subject { Helper.new("007","-j10").run }
+  #  it { should be_success }
+  #  its(:elapsed_time) { should be_within(1).of(2) } # 1..3 sec
+  #end
 
   context "dir=008 --show-conf & PASS_ENV" do
     subject { Helper.new("008","--show-conf ENV1=hoge").run }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 0.9.9.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-30 00:00:00.000000000 Z
+date: 2015-12-11 00:00:00.000000000 Z
 dependencies: []
 description: Parallel workflow extension for Rake
 email:
@@ -16,41 +16,70 @@ email:
 executables:
 - gfwhere-pipe
 - pwrake
+- pwrake_branch
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- CHANGES_V2.md
 - Gemfile
-- LICENSE.txt
+- MIT-LICENSE
+- README
 - README.md
 - Rakefile
 - bin/gfwhere-pipe
 - bin/pwrake
-- lib/pwrake.rb
-- lib/pwrake/application.rb
-- lib/pwrake/counter.rb
-- lib/pwrake/file_utils.rb
-- lib/pwrake/gfarm_feature.rb
-- lib/pwrake/gfwhere_pool.rb
-- lib/pwrake/graphviz.rb
-- lib/pwrake/host_list.rb
-- lib/pwrake/locality_aware_queue.rb
+- bin/pwrake_branch
+- lib/pwrake/branch.rb
+- lib/pwrake/branch/branch.rb
+- lib/pwrake/branch/branch_application.rb
+- lib/pwrake/branch/fiber_queue.rb
+- lib/pwrake/branch/file_utils.rb
+- lib/pwrake/branch/shell.rb
+- lib/pwrake/branch/shell_profiler.rb
+- lib/pwrake/branch/worker_communicator.rb
+- lib/pwrake/gfarm/gfarm_path.rb
+- lib/pwrake/gfarm/gfarm_postprocess.rb
+- lib/pwrake/iomux/channel.rb
+- lib/pwrake/iomux/handler.rb
+- lib/pwrake/iomux/handler_set.rb
+- lib/pwrake/iomux/runner.rb
 - lib/pwrake/logger.rb
 - lib/pwrake/master.rb
-- lib/pwrake/mcgp.rb
-- lib/pwrake/option.rb
-- lib/pwrake/profiler.rb
-- lib/pwrake/rake_modify.rb
+- lib/pwrake/master/fiber_pool.rb
+- lib/pwrake/master/idle_cores.rb
+- lib/pwrake/master/master.rb
+- lib/pwrake/master/master_application.rb
+- lib/pwrake/master/postprocess.rb
+- lib/pwrake/misc/graphviz.rb
+- lib/pwrake/misc/mcgp.rb
+- lib/pwrake/option/host_map.rb
+- lib/pwrake/option/option.rb
+- lib/pwrake/option/option_filesystem.rb
+- lib/pwrake/queue/locality_aware_queue.rb
+- lib/pwrake/queue/no_action_queue.rb
+- lib/pwrake/queue/queue_array.rb
+- lib/pwrake/queue/task_queue.rb
 - lib/pwrake/report.rb
 - lib/pwrake/report/parallelism.rb
 - lib/pwrake/report/report.rb
 - lib/pwrake/report/report_multi.rb
 - lib/pwrake/report/stat.rb
-- lib/pwrake/shell.rb
-- lib/pwrake/task_algorithm.rb
-- lib/pwrake/task_queue.rb
-- lib/pwrake/timer.rb
+- lib/pwrake/report/task_stat.rb
+- lib/pwrake/task/task_algorithm.rb
+- lib/pwrake/task/task_manager.rb
+- lib/pwrake/task/task_property.rb
+- lib/pwrake/task/task_rank.rb
+- lib/pwrake/task/task_wrapper.rb
 - lib/pwrake/version.rb
+- lib/pwrake/worker/executor.rb
+- lib/pwrake/worker/gfarm_directory.rb
+- lib/pwrake/worker/invoker.rb
+- lib/pwrake/worker/load.rb
+- lib/pwrake/worker/log_executor.rb
+- lib/pwrake/worker/shared_directory.rb
+- lib/pwrake/worker/worker_main.rb
+- lib/pwrake/worker/writer.rb
 - pwrake.gemspec
 - setup.rb
 - spec/001/Rakefile
@@ -90,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Adding Parallel and Distributed feature to Rake

data/lib/pwrake.rb

@@ -1,19 +0,0 @@
-require "thread"
-require "pathname"
-
-require "pwrake/version"
-
-require "pwrake/logger"
-require "pwrake/timer"
-require "pwrake/counter"
-require "pwrake/file_utils"
-require "pwrake/profiler"
-require "pwrake/shell"
-require "pwrake/task_algorithm"
-require "pwrake/task_queue"
-require "pwrake/host_list"
-require "pwrake/option"
-require "pwrake/master"
-require "pwrake/application"
-
-require "pwrake/rake_modify"

data/lib/pwrake/application.rb

@@ -1,232 +0,0 @@
-module Rake
-  class << self
-    def application
-      @application ||= Pwrake.application
-    end
-  end
-end
-
-
-module Pwrake
-
-  class << self
-    def application
-      @application ||= Application.new
-    end
-  end
-
-  class Application < ::Rake::Application
-
-    def filesystem
-      @master.filesystem
-    end
-
-    def task_queue
-      @master.task_queue
-    end
-
-    def finish_queue
-      @master.finish_queue
-    end
-
-    def postprocess(t)
-      if @master.postprocess
-        @master.postprocess.postprocess(t)
-      end
-    end
-
-    def thread_loop(*args)
-      @master.thread_loop(*args)
-    end
-
-    def pwrake_options
-      @master.pwrake_options
-    end
-
-    def start_worker
-      @master.start
-    end
-
-    def host_list
-      @master.host_list
-    end
-
-    def task_logger
-      @master.task_logger
-    end
-
-    def task_id_counter
-      @master.task_id_counter
-    end
-
-    # Run the Pwrake application.
-    def run
-      standard_exception_handling do
-        init("pwrake")
-        @master = Master.new
-        @master.init
-        load_rakefile
-        begin
-          begin
-            top_level
-          rescue
-            puts $!.message
-          end
-        ensure
-          @master.finish
-        end
-      end
-    end
-
-    # Run the top level tasks of a Rake application.
-    def top_level
-      standard_exception_handling do
-        if options.show_tasks
-          display_tasks_and_comments
-        elsif options.show_prereqs
-          display_prerequisites
-        else
-          top_level_tasks.each { |task_name| invoke_task(task_name) }
-        end
-      end
-    end
-
-
-    def standard_rake_options
-      opts = super
-      opts.each_with_index do |a,i|
-        if a[0] == '--version'
-          a[3] = lambda { |value|
-            puts "rake, version #{RAKEVERSION}"
-            puts "pwrake, version #{Pwrake::VERSION}"
-            exit
-          }
-        end
-      end
-
-      opts.concat(
-      [
-       ['-F', '--hostfile FILE',
-        "[Pw] Read hostnames from FILE",
-        lambda { |value|
-          options.hostfile = value
-        }
-       ],
-       ['-j', '--jobs [N]',
-        "[Pw] Number of threads at localhost (default: # of processors)",
-        lambda { |value|
-          if value
-            value = value.to_i
-            if value > 0
-              options.num_threads = value
-            else
-              options.num_threads = x = processor_count + value
-              raise "negative/zero number of threads (#{x})" if x <= 0
-            end
-          else
-            options.num_threads = processor_count
-          end
-        }
-       ],
-       ['-L', '--logfile [FILE]', "[Pw] Write log to FILE",
-        lambda { |value|
-          if value.kind_of? String
-            options.logfile = value
-          else
-            options.logfile = ""
-          end
-        }
-       ],
-       ['--ssh-opt', '--ssh-option OPTION', "[Pw] Option passed to SSH",
-        lambda { |value|
-          options.ssh_option = value
-        }
-       ],
-       ['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|gfarm)",
-        lambda { |value|
-          options.filesystem = value
-        }
-       ],
-       ['--gfarm', "[Pw] FILESYSTEM=gfarm",
-        lambda { |value|
-          options.filesystem = "gfarm"
-        }
-       ],
-       ['-A', '--disable-affinity', "[Pw] Turn OFF affinity (AFFINITY=off)",
-        lambda { |value|
-          options.disable_affinity = true
-        }
-       ],
-       ['-S', '--disable-steal', "[Pw] Turn OFF task steal",
-        lambda { |value|
-          options.disable_steal = true
-        }
-       ],
-       ['-d', '--debug',
-        "[Pw] Output Debug messages",
-        lambda { |value|
-          options.debug = true
-        }
-       ],
-       ['--pwrake-conf [FILE]',
-        "[Pw] Pwrake configuation file in YAML",
-        lambda {|value| options.pwrake_conf = value}
-       ],
-       ['--show-conf','--show-config',
-        "[Pw] Show Pwrake configuration options",
-        lambda {|value| options.show_conf = true }
-       ],
-       ['--report LOG', "[Pw] Report profile HTML from LOG and exit.",
-         lambda { |value|
-           require 'pwrake/report'
-           Report.new(File.basename(value.sub(/\.[^.]+$/,"")),[]).report_html
-           exit
-         }
-       ]
-
-      ])
-      opts
-    end
-
-
-    def count(host_list, host)
-      @master.counter.count( host_list, host )
-    end
-
-=begin
-    def host_weigts
-      @master.counter.host_weights
-    end
-=end
-
-    # from Michael Grosser's parallel
-    # https://github.com/grosser/parallel
-    def processor_count
-      host_os = RbConfig::CONFIG['host_os']
-      case host_os
-      when /linux|cygwin/
-        ncpu = 0
-        open("/proc/cpuinfo").each do |l|
-          ncpu += 1 if /^processor\s+: \d+/=~l
-        end
-        ncpu
-      when /darwin9/
-        `hwprefs cpu_count`.to_i
-      when /darwin/
-        (hwprefs_available? ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
-      when /(open|free)bsd/
-        `sysctl -n hw.ncpu`.to_i
-      when /mswin|mingw/
-        require 'win32ole'
-        wmi = WIN32OLE.connect("winmgmts://")
-        cpu = wmi.ExecQuery("select NumberOfLogicalProcessors from Win32_Processor")
-        cpu.to_enum.first.NumberOfLogicalProcessors
-      when /solaris2/
-        `psrinfo -p`.to_i # physical cpus
-      else
-        raise "Unknown architecture: #{host_os}"
-      end
-    end
-
-  end
-end

data/lib/pwrake/counter.rb

@@ -1,54 +0,0 @@
-module Pwrake
-
-  class Counter
-
-    def initialize
-      @same = 0
-      @diff = 0
-      @total = 0
-      @same_hosts = {}
-      @diff_hosts = {}
-      @no_queue = 0
-      @found_queue = 0
-      @empty_queue = 0
-    end
-
-    def count(host_list, host)
-      @total += 1
-      if host_list and host_list.include?(host)
-        @same += 1
-        @same_hosts[host] = (@same_hosts[host]||0) + 1
-      else
-        @diff += 1
-        @diff_hosts[host] = (@diff_hosts[host]||0) + 1
-      end
-    end
-
-    def print
-      s  = "same=#{@same}, diff=#{@diff}, total=#{@total}\n"
-      s << "same_hosts = {\n"
-      @same_hosts.keys.sort.each do |k|
-        s << "  #{k}: #{@same_hosts[k]}\n"
-      end
-      s << "}\n"
-      s << "different_hosts = {\n"
-      @diff_hosts.keys.sort.each do |k|
-        s << "  #{k}: #{@diff_hosts[k]}\n"
-      end
-      s << "}"
-      Log.info s
-    end
-
-    def no_queue
-      @no_queue += 1
-    end
-
-    def found_queue
-      @found_queue += 1
-    end
-
-    def empty_queue
-      @empty_queue += 1
-    end
-  end
-end