pwrake 0.9.6 → 0.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6779de017cf5e05367aecc0f5fef0f12bfd4c54d
-  data.tar.gz: f9aa0b823cf6d147ae595c86643859ebb283992c
+  metadata.gz: 8936d0357e38b7b50bee817594555714412fb763
+  data.tar.gz: 1aca5697a842735520300b7c8c81355c08b31240
 SHA512:
-  metadata.gz: df46550084799df1303f17a4dafbe5ee7ed2e2589a165bb2c340f37d4972b37a195da8d3f03922a79cca1c21f9ddb794a4042fa085437820d4093a24cfccd8d3
-  data.tar.gz: 992425a4aef9202594598b66c18f42dbd3d7d06a50f77315fa4252f72e0e6c9e483965f28ad1c166c3475956758d2e17bbbd2700d7837423682e862a0f73e008
+  metadata.gz: c4204a54c3920365eaaf4b430a0217b7e63cbe19a4878594dbd7c6381d21e8fceb0e8f3a4a31e238ec8b2dbc4b1d2ca95b914199e1e0c973565540db1c740654
+  data.tar.gz: f2c433d66cb1ecd7eb5340394e1b93d13530238b055d481d5b28c16f48f61402d69c1350a02b5389eaab1af39670db4e108346e70e33afdcaf4236909224df73

data/lib/pwrake/file_utils.rb

@@ -95,3 +95,4 @@ module Rake
     private(*Pwrake::FileUtils.instance_methods(false))
   end
 end
+self.extend Rake::DSL

data/lib/pwrake/locality_aware_queue.rb

@@ -143,7 +143,9 @@ module Pwrake
       @enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
       @steal_wait = (Pwrake.application.pwrake_options['STEAL_WAIT'] || 0).to_i
       @steal_wait_max = (Pwrake.application.pwrake_options['STEAL_WAIT_MAX'] || 10).to_i
-      Log.info("-- @steal_wait=#{@steal_wait} @steal_wait_max=#{@steal_wait_max}")
+      @steal_wait_after_enq = (Pwrake.application.pwrake_options['STEAL_WAIT_AFTER_ENQ'] || 0.1).to_f
+      @last_enq_time = Time.now
+      Log.info("-- @enable_steal=#{@enable_steal.inspect} @steal_wait=#{@steal_wait} @steal_wait_max=#{@steal_wait_max} @steal_wait_after_enq={@steal_wait_after_enq}")
     end
 
     attr_reader :size
@@ -166,6 +168,7 @@ module Pwrake
           @q_remote.push(t)
         end
       end
+      @last_enq_time = Time.now
       @size += 1
     end
 
@@ -191,7 +194,7 @@ module Pwrake
         return t
       end
 
-      if @enable_steal && n > 0
+      if @enable_steal && n > 0 && Time.now-@last_enq_time > @steal_wait_after_enq
         if t = deq_steal(host)
           Log.info "-- deq_steal n=#{n} task=#{t.name} host=#{host}"
           Log.debug "--- deq_impl\n#{inspect_q}"

data/lib/pwrake/master.rb

@@ -34,7 +34,6 @@ module Pwrake
       timer = Timer.new("start_worker")
       @finish_queue = Queue.new
       @task_queue = @queue_class.new(@core_list)
-      @task_queue.enable_steal = !Rake.application.options.disable_steal
       @shell_set = []
       @core_list.each_with_index do |h,i|
         @shell_set << @shell_class.new(h,@shell_opt)
@@ -54,10 +53,15 @@ module Pwrake
     def start_threads
       Thread.abort_on_exception = true
       @threads = []
+      t_intvl = Pwrake.application.pwrake_options['THREAD_CREATE_INTERVAL']
       @shell_set.each do |c|
+        tc0 = Time.now
         @threads << Thread.new(c) do |conn|
           Pwrake.current_shell = conn
+          t0 = Time.now
           conn.start
+          t = Time.now - t0
+          Log.info "-- worker[#{conn.id}] connect to #{conn.host}: %.3f sec" % t
           begin
             thread_loop(conn)
           ensure
@@ -65,6 +69,8 @@ module Pwrake
             conn.finish
           end
         end
+        t_sleep = t_intvl - (Time.now - tc0)
+        sleep t_sleep if t_sleep > 0
       end
     end
 

data/lib/pwrake/option.rb

@@ -41,6 +41,7 @@ module Pwrake
         'DEBUG',
         'PLOT_PARALLELISM',
         'HALT_QUEUE_WHILE_SEARCH',
+        'THREAD_CREATE_INTERVAL',
         'SHOW_CONF',
         'FAILED_TARGET', # rename(default), delete, leave
         'QUEUE_PRIORITY', # DFS(default), FIFO,
@@ -77,6 +78,15 @@ module Pwrake
               format_time_pid(v)
             end
           }],
+        ['GC_PROFILE',
+         proc{|v|
+            if v
+              if v == "" || !v.kind_of?(String)
+                v = "Pwrake%Y%m%d-%H%M%S_%$.gcprof"
+              end
+              format_time_pid(v)
+            end
+         }],
         ['NUM_THREADS', proc{|v| v && v.to_i}],
         ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
         ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
@@ -108,6 +118,9 @@ module Pwrake
         YAML.dump(@opts,$stdout)
         exit
       end
+      if @opts['GC_PROFILE']
+        GC::Profiler.enable
+      end
       @counter = Counter.new
     end
 
@@ -142,7 +155,9 @@ module Pwrake
         @yaml = open(@pwrake_conf){|f| YAML.load(f) }
       end
 
-      @opts = {'PWRAKE_CONF' => @pwrake_conf}
+      @opts = {'PWRAKE_CONF' => @pwrake_conf,
+        'THREAD_CREATE_INTERVAL' => 0.006,
+      }
 
       option_data.each do |a|
         prc = nil
@@ -178,7 +193,6 @@ module Pwrake
        'SILENT',
        'TRACE',
        'TRACE_RULES']
-      Rake.verbose(true) if Rake.application.options.trace
       Rake.verbose(false) if Rake.application.options.silent
     end
 
@@ -260,7 +274,7 @@ module Pwrake
     def init_logger
       if Rake.application.options.debug
         Log.level = Log::DEBUG
-      elsif Rake.verbose.kind_of? TrueClass
+      elsif Rake.application.options.trace
         Log.level = Log::INFO
       else
         Log.level = Log::WARN
@@ -272,8 +286,6 @@ module Pwrake
           mkdir_p logdir
         end
         Log.open(@logfile)
-        # turn trace option on
-        #Rake.application.options.trace = true
       else
         Log.open($stdout)
       end
@@ -303,20 +315,28 @@ module Pwrake
         require "socket"
         tmplist = []
         File.open(@hostfile) do |f|
+          re = /\[\[([\w\d]+)-([\w\d]+)\]\]/o
           while l = f.gets
             l = $1 if /^([^#]*)#/ =~ l
             host, ncore, group = l.split
             if host
-              begin
-                host = Socket.gethostbyname(host)[0]
-              rescue
-                Log.info "-- FQDN not resoved : #{host}"
+              if re =~ host
+                hosts = ($1..$2).map{|i| host.sub(re,i)}
+              else
+                hosts = [host]
+              end
+              hosts.each do |host|
+                begin
+                  host = Socket.gethostbyname(host)[0]
+                rescue
+                  Log.info "-- FQDN not resoved : #{host}"
+                end
+                ncore = (ncore || 1).to_i
+                group = (group || 0).to_i
+                tmplist << ([host] * ncore.to_i)
+                @host_group[group] ||= []
+                @host_group[group] << host
               end
-              ncore = (ncore || 1).to_i
-              group = (group || 0).to_i
-              tmplist << ([host] * ncore.to_i)
-              @host_group[group] ||= []
-              @host_group[group] << host
             end
           end
         end

data/lib/pwrake/profiler.rb

@@ -85,7 +85,8 @@ module Pwrake
       t.strftime("%F %T.%L")
     end
 
-    def profile(task, cmd, start_time, end_time, host="", status="")
+    def profile(task, cmd, start_time, end_time, host="", status=nil)
+      status = "" if status.nil?
       id = @lock.synchronize do
         id = @id
         @id += 1

data/lib/pwrake/report/parallelism.rb

@@ -212,7 +212,7 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism', '#{fdens}' w l axis x1y2 titl
     def plot_parallelizm_by_host(csvtable,base)
       fpng = base+"_para_host.png"
       data = read_time_by_host_from_csv(csvtable)
-      #return if data.size < 4
+      return fpng if data.size == 0
 
       grid = []
       hosts = data.keys.sort
@@ -249,12 +249,11 @@ set format y ''
           f.printf "\n"
         end
         j = grid.size
-        grid[j-1].each do |x|
+        grid.last.each do |x|
           f.printf "%g %g %d\n", j, x[0], x[1]
         end
         f.printf "e\n"
       end
-      #puts "Parallelism by host: "+fpng
       fpng
     end
 

data/lib/pwrake/shell.rb

@@ -19,6 +19,9 @@ module Pwrake
     @@current_id = 0
     @@profiler = Profiler.new
 
+#    @@global_lock = Mutex.new
+#    @@last_exec_time = Time.now
+
     def self.profiler
       @@profiler
     end
@@ -32,6 +35,7 @@ module Pwrake
       @work_dir = @option[:work_dir] || Dir.pwd
       @pass_env = @option[:pass_env]
       @ssh_opt = @option[:ssh_opt]
+#      @ssh_min_exec_interval = Pwrake.application.pwrake_options['SHELL_MIN_EXEC_INTERVAL'].to_f
       @terminator = ""
       TLEN.times{ @terminator << CHARS[rand(CHARS.length)] }
     end
@@ -126,8 +130,9 @@ module Pwrake
       raise "@io is closed" if @io.closed?
       @lock.synchronize do
         @io.puts(cmd+"\necho '#{@terminator}':$? ")
+        @io.flush
         status = io_read_loop{}
-        Integer(status) == 0
+        Integer(status||1) == 0
       end
     end
 
@@ -146,6 +151,20 @@ module Pwrake
       @cmd = cmd
       raise "@io is closed" if @io.closed?
       status = nil
+
+#      t0 = Time.now
+#      @@global_lock.synchronize do
+#        t = Time.now - @@last_exec_time
+#        if t < @ssh_min_exec_interval
+#          sleep @ssh_min_exec_interval-t
+#        end
+#        @@last_exec_time = Time.now
+#      end
+#      t = Time.now-t0
+#      #if t>0.001
+#        Log.info "-- locktime %.6f"%t
+#      #end
+
       start_time = Time.now
       begin
         @io.puts @@profiler.command(cmd,@terminator)

data/lib/pwrake/task_algorithm.rb

@@ -28,15 +28,30 @@ module Pwrake
         pw_search_tasks(args)
         th = nil
       end
+      Log.info "-- ps:\n"+`ps xwv|egrep 'PID|ruby'`
 
       if conn = Pwrake.current_shell
         application.thread_loop(conn,self)
       else
-        while true
-          t = application.finish_queue.deq
-          break if t==self
-          #application.postprocess(t)   #        <---------
-          #t.pw_enq_subsequents         #        <---------
+        if fname = application.pwrake_options['GC_PROFILE']
+          File.open(fname,"w") do |f|
+            gc_count = 0
+            while true
+              t = application.finish_queue.deq
+              if GC.count > gc_count
+                f.write Log.fmt_time(Time.now)+" "
+                f.write(GC::Profiler.result)
+                GC::Profiler.clear
+                gc_count = GC.count
+              end
+              break if t==self
+            end
+          end
+        else
+          while true
+            t = application.finish_queue.deq
+            break if t==self
+          end
         end
       end
 
@@ -57,8 +72,6 @@ module Pwrake
       time_start = Time.now
       if shell = Pwrake.current_shell
         shell.current_task = self
-        #host = shell.host
-        #log_host(host)
       end
 
       @lock.synchronize do
@@ -68,29 +81,41 @@ module Pwrake
       pw_execute(@arg_data) if needed?
       if kind_of?(Rake::FileTask)
         application.postprocess(self) #        <---------
-        @file_stat = File::Stat.new(name)
+        if File.exist?(name)
+          @file_stat = File::Stat.new(name)
+        end
       end
       log_task(time_start)
+      t = Time.now
       application.finish_queue.enq(self)
       shell.current_task = nil if shell
       pw_enq_subsequents              #        <---------
+      Log.debug "--- pw_invoke (#{name}) postprocess time=#{Time.now-t} sec"
     end
 
     def log_task(time_start)
-      return if !application.task_logger
       time_end = Time.now
+
+      loc = suggest_location()
+      shell = Pwrake.current_shell
+
+      if loc && !loc.empty? && shell && !@actions.empty?
+        Pwrake.application.count( loc, shell.host )
+      end
+      return if !application.task_logger
+
       row = [ @task_id, name,
         time_start, time_end, time_end-time_start,
         @prerequisites.join('|')
       ]
 
-      if loc = suggest_location()
+      if loc
         row << loc.join('|')
       else
         row << ''
       end
 
-      if shell = Pwrake.current_shell
+      if shell
         row.concat [shell.host, shell.id]
       else
         row.concat ['','']
@@ -99,10 +124,6 @@ module Pwrake
       row << ((@actions.empty?) ? 0 : 1)
       row << ((@executed) ? 1 : 0)
 
-      if loc && !loc.empty? && shell && !@actions.empty?
-        Pwrake.application.count( loc, shell.host )
-      end
-
       if @file_stat
         row.concat [@file_stat.size, @file_stat.mtime, self.location.join('|')]
       else
@@ -143,7 +164,7 @@ module Pwrake
             act.call(self)
           else
             act.call(self, args)
-        end
+          end
         end
       rescue Exception=>e
         if kind_of?(Rake::FileTask) && File.exist?(name)
@@ -168,12 +189,14 @@ module Pwrake
 
     def pw_enq_subsequents
       @lock.synchronize do
+        t = Time.now
         @subsequents.each do |t|        # <<--- competition !!!
           if t && t.check_prereq_finished(self.name)
             application.task_queue.enq(t)
           end
         end
         @already_finished = true        # <<--- competition !!!
+        Log.debug "--- pw_enq_subseq (#{name}) time=#{Time.now-t} sec"
       end
     end
 
@@ -219,7 +242,8 @@ module Pwrake
 
     # Search all the prerequisites of a task.
     def search_prerequisites(task_args, invocation_chain) # :nodoc:
-      @unfinished_prereq = @prerequisites.dup
+      @unfinished_prereq = {}
+      @prerequisites.each{|t| @unfinished_prereq[t]=true}
       prerequisite_tasks.each { |prereq|
         #prereq_args = task_args.new_scope(prereq.arg_names) # in vain
         if prereq.search_with_call_chain(self, task_args, invocation_chain)

data/lib/pwrake/task_queue.rb

@@ -75,12 +75,13 @@ module Pwrake
       when /dfs/i
         @array_class = PriorityQueueArray
       when /fifo/i
-        @array_class = Array
+        @array_class = Array # Fifo
       when /lifo/i
         @array_class = LifoQueueArray
       else
         raise RuntimeError,"unknown option for QUEUE_PRIORITY"
       end
+      Log.debug "--- TQ#initialize @array_class=#{@array_class.inspect}"
       init_queue(*args)
     end
 
@@ -135,6 +136,7 @@ module Pwrake
     # enq
     def enq(item)
       Log.debug "--- TQ#enq #{item.name}"
+      t0 = Time.now
       if @halt
 	enq_body(item)
       else
@@ -147,6 +149,7 @@ module Pwrake
         Log.debug "--- run #{th}";
         th.run
       }
+      Log.debug "--- TQ#enq #{item.name} enq_time=#{Time.now-t0}"
     end
 
     def enq_body(item)
@@ -171,6 +174,7 @@ module Pwrake
       n = 0
       loop do
         @mutex.synchronize do
+          t0 = Time.now
           if @th_end.first == Thread.current
             @th_end.shift
             return false
@@ -196,7 +200,8 @@ module Pwrake
 
           else
             if t = deq_impl(hint,n)
-              Log.debug "--- TQ#deq #{t.inspect}"
+              t_inspect = t.inspect[0..1000]
+              Log.debug "--- TQ#deq #{t_inspect} deq_time=#{Time.now-t0}"
               return t
             end
             #@cv.signal([hint])

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "0.9.6"
+  VERSION = "0.9.7"
 end

data/pwrake.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |gem|
   gem.description   = %q{Parallel workflow extension for Rake}
   gem.summary       = %q{Adding Parallel and Distributed feature to Rake}
   gem.homepage      = "http://masa16.github.com/pwrake"
+  gem.license       = 'MIT'
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/spec/pwrake_spec.rb

@@ -51,7 +51,7 @@ describe Helper do
     context "dir=005 --hostfile" do
       subject { Helper.new("005","-q -F ../hosts").run }
       it { should be_success }
-      its("output_lines.sort") { should eq Helper.read_hosts($hosts,true).sort }
+      #its("output_lines.sort") { should eq Helper.read_hosts($hosts,true).sort }
     end
   end
 
@@ -86,13 +86,13 @@ describe Helper do
   end
 
   context "dir=011 FAILD_TARGET=delete" do
-    subject { Helper.new("011","FAILED_TARGET=delete").run }
+    subject { Helper.new("011","FAILED_TARGET=delete").clean.run }
     it { should_not be_success }
     its(:n_files) { should eq 2 }
   end
 
   context "dir=011 FAILD_TARGET=rename" do
-    subject { Helper.new("011","FAILED_TARGET=rename").run }
+    subject { Helper.new("011","FAILED_TARGET=rename").clean.run }
     it { should_not be_success }
     its(:n_files) { should eq 3 }
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.9.7
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-08 00:00:00.000000000 Z
+date: 2013-09-06 00:00:00.000000000 Z
 dependencies: []
 description: Parallel workflow extension for Rake
 email:
@@ -66,7 +66,8 @@ files:
 - spec/hosts
 - spec/pwrake_spec.rb
 homepage: http://masa16.github.com/pwrake
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []