pwrake 0.9.5 → 0.9.6

data/lib/pwrake/shell.rb

@@ -36,7 +36,7 @@ module Pwrake
       TLEN.times{ @terminator << CHARS[rand(CHARS.length)] }
     end
 
-    attr_reader :id
+    attr_reader :id, :host
     attr_accessor :current_task
 
     def system_cmd(*arg)
@@ -95,18 +95,22 @@ module Pwrake
       command = command.join(' ')
       Log.debug "--- command=#{command.inspect}"
       @lock.synchronize do
-        _execute(command){|x| LOCK.synchronize{puts x}}
+        _execute(command){|x| Log.stdout_puts x}
       end
+      @status == 0
     end
 
     def cd_work_dir
-      _system("cd #{@work_dir}")
+      _system("cd #{@work_dir}") or die
     end
 
     def cd(dir="")
-      _system("cd #{dir}")
+      _system("cd #{dir}") or die
     end
 
+    def die
+      raise "Failed at #{@host}, id=#{@id}, cmd='#{@cmd}'"
+    end
 
     END {
       OPEN_LIST.map do |k,v|
@@ -118,6 +122,7 @@ module Pwrake
     private
 
     def _system(cmd)
+      @cmd = cmd
       raise "@io is closed" if @io.closed?
       @lock.synchronize do
         @io.puts(cmd+"\necho '#{@terminator}':$? ")
@@ -126,15 +131,30 @@ module Pwrake
       end
     end
 
+    def _backquote(cmd)
+      @cmd = cmd
+      raise "@io is closed" if @io.closed?
+      a = []
+      @lock.synchronize do
+        @io.puts(cmd+"\necho '#{@terminator}':$? ")
+        status = io_read_loop{|x| a << x}
+      end
+      a.join("\n")
+    end
+
     def _execute(cmd,quote=nil,&block)
+      @cmd = cmd
       raise "@io is closed" if @io.closed?
+      status = nil
       start_time = Time.now
-      @io.puts @@profiler.command(cmd,@terminator)
-      status = io_read_loop(&block)
-      end_time = Time.now
-      @status = @@profiler.profile(@current_task, cmd,
-                                   start_time, end_time, status)
-      @status == 0
+      begin
+        @io.puts @@profiler.command(cmd,@terminator)
+        status = io_read_loop(&block)
+      ensure
+        end_time = Time.now
+        @status = @@profiler.profile(@current_task, cmd,
+                                     start_time, end_time, host, status)
+      end
     end
 
     def io_read_loop

data/lib/pwrake/task_algorithm.rb

@@ -13,50 +13,115 @@ module Pwrake
       @location = a
     end
 
+    def task_id
+      @task_id
+    end
 
     def invoke_modify(*args)
+      return if @already_invoked
+
       application.start_worker
-      task_args = TaskArguments.new(arg_names, args)
-      flag = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
-      start_time = Time.now
-      if flag
-        application.task_queue.enq_synchronize do
-          search_with_call_chain(self, task_args, InvocationChain::EMPTY)
-        end
+
+      if false
+        th = Thread.new(args){|a| pw_search_tasks(a) }
       else
-        search_with_call_chain(self, task_args, InvocationChain::EMPTY)
+        pw_search_tasks(args)
+        th = nil
       end
-      Log.info "-- search_tasks %.6fs" % (Time.now-start_time)
-      return if @already_invoked
 
       if conn = Pwrake.current_shell
-        @waiting_thread = nil
         application.thread_loop(conn,self)
       else
-        @waiting_thread = Thread.current
-        Log.debug "--- sleep in invoke_modify #{self.name} #{@waiting_thread.inspect}"
-        sleep
-        Log.debug "--- awake in invoke_modify"
-        pw_invoke
+        while true
+          t = application.finish_queue.deq
+          break if t==self
+          #application.postprocess(t)   #        <---------
+          #t.pw_enq_subsequents         #        <---------
+        end
       end
+
+      th.join if th
     end
 
+    def pw_search_tasks(args)
+      task_args = TaskArguments.new(arg_names, args)
+      timer = Timer.new("search_task")
+      h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
+      application.task_queue.synchronize(h) do
+	search_with_call_chain(self, task_args, InvocationChain::EMPTY)
+      end
+      timer.finish
+    end
 
     def pw_invoke
-      shell = Pwrake.current_shell
-      if shell
+      time_start = Time.now
+      if shell = Pwrake.current_shell
         shell.current_task = self
-        log_host(shell.host)
+        #host = shell.host
+        #log_host(host)
       end
+
       @lock.synchronize do
         return if @already_invoked
         @already_invoked = true
       end
       pw_execute(@arg_data) if needed?
-      start_time = Time.now
-      pw_enq_subsequents
-      Log.info "-- pw_enq_subsequents(%s) %.6fs"%[self.name,Time.now-start_time]
+      if kind_of?(Rake::FileTask)
+        application.postprocess(self) #        <---------
+        @file_stat = File::Stat.new(name)
+      end
+      log_task(time_start)
+      application.finish_queue.enq(self)
       shell.current_task = nil if shell
+      pw_enq_subsequents              #        <---------
+    end
+
+    def log_task(time_start)
+      return if !application.task_logger
+      time_end = Time.now
+      row = [ @task_id, name,
+        time_start, time_end, time_end-time_start,
+        @prerequisites.join('|')
+      ]
+
+      if loc = suggest_location()
+        row << loc.join('|')
+      else
+        row << ''
+      end
+
+      if shell = Pwrake.current_shell
+        row.concat [shell.host, shell.id]
+      else
+        row.concat ['','']
+      end
+
+      row << ((@actions.empty?) ? 0 : 1)
+      row << ((@executed) ? 1 : 0)
+
+      if loc && !loc.empty? && shell && !@actions.empty?
+        Pwrake.application.count( loc, shell.host )
+      end
+
+      if @file_stat
+        row.concat [@file_stat.size, @file_stat.mtime, self.location.join('|')]
+      else
+        row.concat ['','','']
+      end
+
+      s = row.map do |x|
+        if x.kind_of?(Time)
+          Profiler.format_time(x)
+        elsif x.kind_of?(String) && x!=''
+          '"'+x+'"'
+        else
+          x.to_s
+        end
+      end.join(',')
+
+      # task_id task_name start_time end_time elap_time preq preq_host
+      # exec_host shell_id has_action executed file_size file_mtime file_host
+      application.task_logger.print s+"\n"
     end
 
 
@@ -71,41 +136,53 @@ module Pwrake
         Log.info "** Execute #{name}"
       end
       application.enhance_with_matching_rule(name) if @actions.empty?
-      @actions.each do |act|
-        case act.arity
-        when 1
-          act.call(self)
-        else
-          act.call(self, args)
+      begin
+        @actions.each do |act|
+          case act.arity
+          when 1
+            act.call(self)
+          else
+            act.call(self, args)
+        end
+        end
+      rescue Exception=>e
+        if kind_of?(Rake::FileTask) && File.exist?(name)
+          opt = application.pwrake_options['FAILED_TARGET']||"rename"
+          case opt
+          when /rename/i
+            dst = name+"._fail_"
+            ::FileUtils.mv(name,dst)
+            msg = "Rename failed target file '#{name}' to '#{dst}'"
+            Log.stderr_puts(msg)
+          when /delete/i
+            ::FileUtils.rm(name)
+            msg = "Delete failed target file '#{name}'"
+            Log.stderr_puts(msg)
+          when /leave/i
+          end
         end
+        raise e
       end
+      @executed = true if !@actions.empty?
     end
 
     def pw_enq_subsequents
       @lock.synchronize do
-        my_name = self.name
-        application.task_queue.enq_synchronize do
-          @subsequents.each do |t|        # <<--- competition !!!
-            t && t.check_and_enq(my_name)
+        @subsequents.each do |t|        # <<--- competition !!!
+          if t && t.check_prereq_finished(self.name)
+            application.task_queue.enq(t)
           end
-          @already_finished = true        # <<--- competition !!!
         end
+        @already_finished = true        # <<--- competition !!!
       end
     end
 
-    def check_and_enq(preq_name=nil)
+    def check_prereq_finished(preq_name=nil)
       @unfinished_prereq.delete(preq_name)
-      if @unfinished_prereq.empty?
-        if @waiting_thread
-          Log.debug "--- @waiting_thread.wakeup name=#{self.name} "
-          @waiting_thread.wakeup
-        else
-          Log.debug "--- check_and_enq enq name=#{self.name} "
-          application.task_queue.enq(self)
-        end
-      end
+      @unfinished_prereq.empty?
     end
 
+
     # Same as search, but explicitly pass a call chain to detect
     # circular dependencies.
     def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
@@ -122,7 +199,16 @@ module Pwrake
         if ! @already_searched
           @already_searched = true
           @arg_data = task_args
-          search_prerequisites(task_args, new_chain)
+          if @prerequisites.empty?
+            @unfinished_prereq = {}
+          else
+            search_prerequisites(task_args, new_chain)
+          end
+          @task_id = application.task_id_counter
+          #check_and_enq
+          if @unfinished_prereq.empty?
+            application.task_queue.enq(self)
+          end
         end
         return false
       end
@@ -140,7 +226,6 @@ module Pwrake
           @unfinished_prereq.delete(prereq.name)
         end
       }
-      check_and_enq
     end
 
     # Format the trace flags for display.
@@ -153,18 +238,57 @@ module Pwrake
     end
     private :format_search_flags
 
+    def file_size
+      @file_stat ? @file_stat.size : 0
+    end
+
+    def prior?
+      kind_of?(Rake::FileTask) && !@prerequisites.empty?
+    end
+
+    def suggest_location
+      if prior? && @suggest_location.nil?
+        @suggest_location = []
+        loc_fsz = Hash.new(0)
+        @prerequisites.each do |preq|
+          t = application[preq]
+          loc = t.location
+          fsz = t.file_size
+          if loc && fsz > 0
+            loc.each do |h|
+              loc_fsz[h] += fsz
+            end
+          end
+        end
+        if !loc_fsz.empty?
+          half_max_fsz = loc_fsz.values.max / 2
+          Log.debug "--- loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
+          loc_fsz.each do |h,sz|
+            if sz > half_max_fsz
+              @suggest_location << h
+            end
+          end
+        end
+      end
+      @suggest_location
+    end
+
+    def suggest_location2
+      if kind_of?(Rake::FileTask) && preq_name = @prerequisites[0]
+        application[preq_name].location
+      end
+    end
 
     def log_host(exec_host)
       # exec_host = Pwrake.current_shell.host
-      prereq_name = @prerequisites[0]
-      if kind_of?(Rake::FileTask) and prereq_name
-        Pwrake.application.count( @location, exec_host )
-        if @location and @location.include? exec_host
+      if loc = suggest_location()
+        Pwrake.application.count( loc, exec_host )
+        if loc.include? exec_host
           compare = "=="
         else
           compare = "!="
         end
-        Log.info "-- access to #{prereq_name}: file_host=#{@location.inspect} #{compare} exec_host=#{exec_host}"
+        Log.info "-- access to #{@prerequisites[0]}: file_host=#{loc.inspect} #{compare} exec_host=#{exec_host}"
       end
     end
 

data/lib/pwrake/task_queue.rb

@@ -1,17 +1,93 @@
 module Pwrake
 
+  class TaskConditionVariable < ConditionVariable
+    def signal(hint=nil)
+      super()
+    end
+  end
+
+  class PriorityQueueArray < Array
+    def push(t)
+      task_id = t.task_id
+      if empty? || last.task_id <= task_id
+        super(t)
+      elsif first.task_id > task_id
+        unshift(t)
+      else
+        lower = 0
+        upper = size-1
+        while lower+1 < upper
+          mid = ((lower + upper) / 2).to_i
+          if self[mid].task_id <= task_id
+            lower = mid
+          else
+            upper = mid
+          end
+        end
+        insert(upper,t)
+      end
+    end
+
+    def index(t)
+      if size < 40
+        return super(t)
+      end
+      task_id = t.task_id
+      if last.task_id < task_id || first.task_id > task_id
+        nil
+      else
+        lower = 0
+        upper = size-1
+        while lower+1 < upper
+          mid = ((lower + upper) / 2).to_i
+          if self[mid].task_id < task_id
+            lower = mid
+          else
+            upper = mid
+          end
+        end
+        mid = upper
+        if self[mid].task_id == task_id
+          Log.debug "--- TQA#index=#{mid}, task_id=#{task_id}"
+          mid
+        end
+      end
+    end
+  end # PriorityQueueArray
+
+  class LifoQueueArray < Array
+    def shift
+      pop
+    end
+  end
+
   class TaskQueue
 
     def initialize(*args)
       @finished = false
       @halt = false
       @mutex = Mutex.new
-      @cv = ConditionVariable.new
       @th_end = []
       @enable_steal = true
-      @q = []
       @reservation = {}
       @reserved_q = {}
+      case Pwrake.application.pwrake_options['QUEUE_PRIORITY']||"DFS"
+      when /dfs/i
+        @array_class = PriorityQueueArray
+      when /fifo/i
+        @array_class = Array
+      when /lifo/i
+        @array_class = LifoQueueArray
+      else
+        raise RuntimeError,"unknown option for QUEUE_PRIORITY"
+      end
+      init_queue(*args)
+    end
+
+    def init_queue(*args)
+      @cv = TaskConditionVariable.new
+      @q_prior = @array_class.new
+      @q_later = Array.new
     end
 
     attr_reader :mutex
@@ -34,15 +110,20 @@ module Pwrake
       end
     end
 
-    def enq_synchronize
-      if @halt
-        ret = yield
-      else
-        @mutex.synchronize do
+    def synchronize(condition)
+      ret = nil
+      if condition
+        @mutex.lock
+	@halt = true
+	begin
           ret = yield
-          enq_finish
+	  @cv.broadcast
+	ensure
+	  @halt = false
+	  @mutex.unlock
         end
-        @cv.broadcast
+      else
+	ret = yield
       end
       @reserved_q.keys.each do |th|
         Log.debug "--- run #{th}";
@@ -51,36 +132,15 @@ module Pwrake
       ret
     end
 
-    def enq_finish
-    end
-
-
-    def enq(item,hint=nil)
-      if th = @reservation[item]
-        @reserved_q[th] = item
-      else
-        enq_impl(item,hint)
-      end
-    end
-
-
-    def enq_bak(item,hint=nil)
-      # Log.debug "--- #{self.class}#enq #{item.inspect}"
-      th = nil
+    # enq
+    def enq(item)
+      Log.debug "--- TQ#enq #{item.name}"
       if @halt
-        if th = @reservation[item]
-          @reserved_q[th] = item
-        else
-          enq_impl(item,hint)
-        end
+	enq_body(item)
       else
         @mutex.synchronize do
-          if th = @reservation[item]
-            @reserved_q[th] = item
-          else
-            enq_impl(item,hint)
-            @cv.signal
-          end
+	  enq_body(item)
+	  @cv.signal(item.suggest_location)
         end
       end
       @reserved_q.keys.each{|th|
@@ -89,13 +149,25 @@ module Pwrake
       }
     end
 
-    def enq_impl(item,hint)
-      @q.push(item)          # FIFO Queue
+    def enq_body(item)
+      if th = @reservation[item]
+	@reserved_q[th] = item
+      else
+	enq_impl(item)
+      end
     end
 
+    def enq_impl(item)
+      if item.prior?
+        @q_prior.push(item)
+      else
+        @q_later.push(item)
+      end
+    end
 
+
+    # deq
     def deq(hint=nil)
-      # Log.debug "--- #{self.class}#deq #{self.inspect}"
       n = 0
       loop do
         @mutex.synchronize do
@@ -104,7 +176,7 @@ module Pwrake
             return false
 
           elsif @halt
-            Log.debug "--- halt in #{self.class}#deq @q=#{@q.inspect}"
+            Log.debug "--- halt in TQ#deq @q=#{@q.inspect}"
             @cv.wait(@mutex)
             n = 0
 
@@ -113,43 +185,47 @@ module Pwrake
             return item
 
           elsif empty? # no item in queue
-            Log.debug "--- empty=true in #{self.class}#deq @finished=#{@finished.inspect}"
+            #Log.debug "--- empty=true in #{self.class}#deq @finished=#{@finished.inspect}"
             if @finished
-              @cv.signal
+	      @cv.signal
               return false
             end
-            Log.debug "--- waiting in #{self.class}#deq @finished=#{@finished.inspect}"
+            #Log.debug "--- waiting in #{self.class}#deq @finished=#{@finished.inspect}"
             @cv.wait(@mutex)
             n = 0
 
           else
             if t = deq_impl(hint,n)
-              Log.debug "--- #{self.class}#deq #{t.inspect}"
+              Log.debug "--- TQ#deq #{t.inspect}"
               return t
             end
+            #@cv.signal([hint])
             n += 1
           end
         end
+        #Thread.pass
       end
     end
 
     def deq_impl(hint,n)
-      @q.shift               # FIFO Queue
+      Log.debug "--- TQ#deq_impl #{@q.inspect}"
+      @q_prior.shift || @q_later.shift
     end
 
     def clear
-      @q.clear
+      @q_prior.clear
+      @q_later.clear
       @reserved_q.clear
     end
 
     def empty?
-      @q.empty? && @reserved_q.empty?
+      @q_prior.empty? && @q_later.empty? && @reserved_q.empty?
     end
 
     def finish
-      Log.debug "--- #{self.class}#finish"
+      Log.debug "--- TQ#finish"
       @finished = true
-      @cv.signal
+      @cv.broadcast
     end
 
     def stop
@@ -163,6 +239,10 @@ module Pwrake
       @th_end.push(th)
       @cv.broadcast
     end
-  end
 
+    def after_check(tasks)
+      # implimented at subclass
+    end
+
+  end
 end

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "0.9.5"
+  VERSION = "0.9.6"
 end

data/spec/004/Rakefile

@@ -1,5 +1,5 @@
 A = 4.times.map do |i|
-  file "A#{i}" do |t|
+  task "A#{i}" do |t|
     sh "sleep 1"
   end.name
 end

data/spec/009/pwrake_conf.yaml

@@ -1,2 +1,3 @@
 PROFILE : true
 GNU_TIME : true
+#TASKLOG : true

data/spec/011/Rakefile

@@ -0,0 +1,15 @@
+require "rake/clean"
+
+hoge = %w[hoge1 hoge2]
+CLEAN.include hoge
+CLEAN.include "*._fail_"
+
+task :default => hoge
+
+file "hoge1" do
+  sh "echo hoge1>hoge1"
+end
+
+file "hoge2" do
+  sh "echo hoge2>hoge2; ls hogehoge"
+end

data/spec/helper.rb

@@ -69,11 +69,15 @@ class Helper
       while l = f.gets
         l = $1 if /^([^#]*)#/ =~ l
         host, ncore, group = l.split
-        if host
+        case host
+        when /^localhost(\.localdomain)?$/
+          host = `hostname`.chomp if ssh
+        when nil
+        else
           host = `ssh #{host} hostname`.chomp if ssh
-          ncore = (ncore || 1).to_i
-          cores.concat( [host] * ncore )
         end
+        ncore = (ncore || 1).to_i
+        cores.concat( [host] * ncore )
       end
     end
     cores