pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/task/task_algorithm.rb

@@ -0,0 +1,107 @@
+module Pwrake
+
+  InvocationChain = Rake::InvocationChain
+  TaskArguments = Rake::TaskArguments
+
+  module TaskAlgorithm
+
+    attr_reader :wrapper
+    attr_reader :subsequents
+    attr_reader :arguments
+    attr_reader :property
+
+    def pw_search_tasks(args)
+      Log.debug "#{self.class}#pw_search_tasks start, args=#{args.inspect}"
+      tm = Time.now
+      task_args = TaskArguments.new(arg_names, args)
+      #timer = Timer.new("search_task")
+      #h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
+      #application.task_queue.synchronize(h) do
+	search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
+      #end
+      #timer.finish
+      Log.debug "#{self.class}#pw_search_tasks end #{Time.now-tm}"
+    end
+
+    # Same as search, but explicitly pass a call chain to detect
+    # circular dependencies.
+    def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
+      new_chain = InvocationChain.append(self, invocation_chain)
+      @lock.synchronize do
+        if application.options.trace
+          #Log.info "** Search #{name} #{format_search_flags}"
+          application.trace "** Search #{name} #{format_search_flags}"
+        end
+
+        return true if @already_finished # <<--- competition !!!
+        @subsequents ||= []
+        @subsequents << subseq if subseq # <<--- competition !!!
+
+        if ! @already_searched
+          @already_searched = true
+          @arguments = task_args
+          @wrapper = TaskWrapper.new(self,task_args)
+          if @prerequisites.empty?
+            @unfinished_prereq = {}
+          else
+            search_prerequisites(task_args, new_chain)
+          end
+          #check_and_enq
+          if @unfinished_prereq.empty?
+            application.task_queue.enq(@wrapper)
+          end
+        end
+        return false
+      end
+    rescue Exception => ex
+      add_chain_to(ex, new_chain)
+      raise ex
+    end
+
+    # Search all the prerequisites of a task.
+    def search_prerequisites(task_args, invocation_chain) # :nodoc:
+      @unfinished_prereq = {}
+      @prerequisites.each{|t| @unfinished_prereq[t]=true}
+      prerequisite_tasks.each { |prereq|
+        prereq_args = task_args.new_scope(prereq.arg_names)
+        if prereq.search_with_call_chain(self, prereq_args, invocation_chain)
+          @unfinished_prereq.delete(prereq.name)
+        end
+      }
+    end
+
+    # Format the trace flags for display.
+    def format_search_flags
+      flags = []
+      flags << "finished" if @already_finished
+      flags << "first_time" unless @already_searched
+      flags << "not_needed" unless needed?
+      flags.empty? ? "" : "(" + flags.join(", ") + ")"
+    end
+    private :format_search_flags
+
+    def pw_enq_subsequents
+      t = Time.now
+      #h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
+      #application.task_queue.synchronize(h) do
+        @subsequents.each do |t|        # <<--- competition !!!
+          if t && t.check_prereq_finished(self.name)
+            application.task_queue.enq(t.wrapper)
+          end
+        end
+      #end
+      @already_finished = true        # <<--- competition !!!
+    end
+
+    def check_prereq_finished(preq_name=nil)
+      @unfinished_prereq.delete(preq_name)
+      @unfinished_prereq.empty?
+    end
+
+    def pw_set_property(property)
+      @property = property
+      self
+    end
+
+  end
+end

data/lib/pwrake/task/task_manager.rb

@@ -0,0 +1,32 @@
+module Pwrake
+
+  module TaskManager
+
+    def initialize
+      @property_by_block = {}
+      @last_property = TaskProperty.new
+      super
+    end
+
+    def last_description=(description)
+      @last_property.parse_description(description)
+      super
+    end
+
+    def define_task(task_class, *args, &block) # :nodoc:
+      prop = @property_by_block[block.object_id]
+      if prop.nil?
+        prop = @last_property
+        @last_property = TaskProperty.new
+      end
+      super.pw_set_property(prop)
+    end
+
+    def create_rule(*args, &block) # :nodoc:
+      @property_by_block[block.object_id] = @last_property
+      @last_property = TaskProperty.new
+      super
+    end
+
+  end
+end

data/lib/pwrake/task/task_property.rb

@@ -0,0 +1,98 @@
+module Pwrake
+
+  class TaskProperty
+
+    attr_reader :ncore, :exclusive, :allow, :deny, :order_allow_deny,
+      :disable_steal
+
+    def parse_description(description)
+      if /\bn_?cores?[=:]\s*([+-]?\d+)/i =~ description
+        @ncore = $1.to_i
+      end
+      if /\bexclusive[=:]\s*(\S+)/i =~ description
+        if /^(y|t)/i =~ $1
+          @exclusive = true
+        end
+      end
+      if /\ballow[=:]\s*(\S+)/i =~ description
+        @allow = $1
+      end
+      if /\bdeny[=:]\s*(\S+)/i =~ description
+        @deny = $1
+      end
+      if /\border[=:]\s*(\S+)/i =~ description
+        case $1
+        when /allow,deny/i
+          @order_allow_deny = true
+        when /deny,allow/i
+          @order_allow_deny = false
+        end
+      end
+      if /\bsteal[=:]\s*(\S+)/i =~ description
+        if /^(n|f)/i =~ $1
+          @disable_steal = true
+        end
+      end
+    end
+
+    def acceptable_for(host_info)
+      if @disable_steal && host_info.steal_flag
+        #Log.debug("@disable_steal && host_info.steal_flag")
+        return false
+      end
+      ncore = (@exclusive) ? 0 : (@ncore || 1)
+      if ncore > 0
+        return false if ncore > host_info.idle_cores
+      else
+        n = host_info.ncore + ncore
+        return false if n < 1 || n > host_info.idle_cores
+      end
+      hn = host_info.name
+      if @allow
+        if @deny
+          if @order_allow_deny
+            return false if !File.fnmatch(@allow,hn) || File.fnmatch(@deny,hn)
+          else
+            return false if File.fnmatch(@deny,hn) && !File.fnmatch(@allow,hn)
+          end
+        else
+          return false if !File.fnmatch(@allow,hn)
+        end
+      else
+        if @deny
+          return false if File.fnmatch(@deny,hn)
+        end
+      end
+      return true
+    end
+
+    def n_used_cores(host_info=nil)
+      nc_node = host_info && host_info.ncore
+      if @ncore.nil?
+        return 1
+      elsif @ncore > 0
+        if nc_node && @ncore > nc_node
+          m = "ncore=#{@ncore} must be <= nc_node=#{nc_node}"
+          Log.fatal m
+          raise RuntimeError,m
+        end
+        return @ncore
+      else
+        if nc_node.nil?
+          m = "host_info.ncore is not set"
+          Log.fatal m
+          raise RuntimeError,m
+        end
+        n = @ncore + nc_node
+        if n > 0
+          return n
+        else
+          m = "ncore+nc_node=#{n} must be > 0"
+          Log.fatal m
+          raise RuntimeError,m
+        end
+      end
+    end
+
+  end
+end

data/lib/pwrake/task/task_rank.rb

@@ -0,0 +1,48 @@
+module Pwrake
+
+  class RankStat
+
+    def initialize
+      @lock = Mutex.new
+      @stat = []
+    end
+
+    def add_sample(rank,elap)
+      @lock.synchronize do
+        stat = @stat[rank]
+        if stat.nil?
+          @stat[rank] = stat = [0,0.0]
+        end
+        stat[0] += 1
+        stat[1] += elap
+        #Log.debug "add_sample rank=#{rank} stat=#{stat.inspect} weight=#{stat[0]/stat[1]}"
+      end
+    end
+
+    def rank_weight
+      @lock.synchronize do
+        sum = 0.0
+        count = 0
+        weight = @stat.map do |stat|
+          if stat
+            w = stat[0]/stat[1]
+            sum += w
+            count += 1
+            w
+          else
+            nil
+          end
+        end
+        if count == 0
+          avg = 1.0
+        else
+          avg = sum/count
+        end
+        [weight, avg]
+      end
+    end
+  end
+
+  RANK_STAT = RankStat.new
+
+end

data/lib/pwrake/task/task_wrapper.rb

@@ -0,0 +1,296 @@
+require 'forwardable'
+
+module Pwrake
+
+  class TaskWrapper
+    extend Forwardable
+
+    @@current_id = 1
+    @@task_logger = nil
+
+    def initialize(task,task_args=nil)
+      @task = task
+      @task_args = task_args
+      @property = task.property
+      @task_id = @@current_id
+      @@current_id += 1
+      @location = []
+      @group = []
+      @group_id
+      @suggest_location = nil
+      @file_stat
+      @input_file_size
+      @input_file_mtime
+      @rank
+      @priority
+      @lock_rank = Monitor.new
+      @executed = false
+      @assigned = []
+      @exec_host = nil
+    end
+
+    def_delegators :@task, :name, :actions, :prerequisites, :subsequents
+    def_delegators :@property, :acceptable_for
+
+    attr_reader :task, :task_id, :group, :group_id, :file_stat
+    attr_reader :location
+    attr_reader :assigned
+    attr_accessor :executed
+    attr_accessor :exec_host
+    attr_accessor :shell_id, :status
+
+    def self.format_time(t)
+      t.strftime("%F %T.%L")
+    end
+
+    def self.init_task_logger(option)
+      if dir = option['LOG_DIR']
+        fn = File.join(dir,option['TASK_CSV_FILE'])
+        @@task_logger = CSV.open(fn,'w')
+        @@task_logger.puts %w[
+          task_id task_name start_time end_time elap_time preq preq_host
+          exec_host shell_id has_action executed file_size file_mtime file_host
+        ]
+      end
+    end
+
+    def self.close_task_logger
+      @@task_logger.close if @@task_logger
+    end
+
+    def preprocess
+      if @shell = Pwrake::Shell.current
+        @shell.current_task = self
+      end
+      @time_start = Time.now
+    end
+
+    def postprocess(location)
+      @executed = true if !@task.actions.empty?
+      tm_taskend = Time.now
+      if is_file_task?
+        t = Time.now
+        if File.exist?(name)
+          @file_stat = File::Stat.new(name)
+          @location = location
+        end
+      end
+      #Log.debug "postprocess time=#{Time.now-tm_taskend}"
+      log_task
+      @shell.current_task = nil if @shell
+      if @status=="end"
+        @task.pw_enq_subsequents
+      end
+    end
+
+    def log_task
+      @time_end = Time.now
+      #
+      loc = suggest_location()
+      shell = Pwrake::Shell.current
+      #
+      if loc && !loc.empty? && shell && !actions.empty?
+        Rake.application.count( loc, shell.host )
+      end
+      return if !@@task_logger
+      #
+      elap = @time_end - @time_start
+      if has_output_file?
+        RANK_STAT.add_sample(rank,elap)
+      end
+      #
+      if @file_stat
+        fstat = [@file_stat.size, @file_stat.mtime, self.location.join('|')]
+      else
+        fstat = [nil]*3
+      end
+      #
+      # task_id task_name start_time end_time elap_time preq preq_host
+      # exec_host shell_id has_action executed file_size file_mtime file_host
+      #
+      row = [ @task_id, name, @time_start, @time_end, elap,
+              prerequisites, loc, @exec_host, @shell_id,
+              (actions.empty?) ? 0 : 1,
+              (@executed) ? 1 : 0,
+              *fstat ]
+      row.map!{|x|
+        if x.kind_of?(Time)
+          TaskWrapper.format_time(x)
+        elsif x.kind_of?(Array)
+          if x.empty?
+            nil
+          else
+            x.join('|')
+          end
+        else
+          x
+        end
+      }
+      @@task_logger << row
+      #
+      clsname = @task.class.to_s.sub(/^(Rake|Pwrake)::/o,"")
+      msg = '%s:"%s" %s: id=%d elap=%.6f exec_host=%s' %
+        [clsname,name,@status,@task_id,elap,@exec_host]
+      if @status=="end"
+        Log.info msg
+      else
+        Log.error msg
+      end
+    end
+
+    def is_file_task?
+      @task.kind_of?(Rake::FileTask)
+    end
+
+    def has_output_file?
+      is_file_task? && !actions.empty?
+    end
+
+    def has_input_file?
+      is_file_task? && !prerequisites.empty?
+    end
+
+    def has_action?
+      !@task.actions.empty?
+    end
+
+    def location=(a)
+      @location = a
+      @group = []
+      #@location.each do |host|
+      #  @group |= [Rake.application.host_list.host2group[host]]
+      #end
+    end
+
+    def suggest_location=(a)
+      @suggest_location = a
+    end
+
+    def suggest_location
+      if has_input_file? && @suggest_location.nil?
+        @suggest_location = []
+        loc_fsz = Hash.new(0)
+        prerequisites.each do |preq|
+          t = Rake.application[preq].wrapper
+          loc = t.location
+          fsz = t.file_size
+          if loc && fsz > 0
+            loc.each do |h|
+              loc_fsz[h] += fsz
+            end
+          end
+        end
+        #Log.debug "input=#{prerequisites.join('|')}"
+        if !loc_fsz.empty?
+          half_max_fsz = loc_fsz.values.max / 2
+          Log.debug "loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
+          loc_fsz.each do |h,sz|
+            if sz > half_max_fsz
+              @suggest_location << h
+            end
+          end
+        end
+      end
+      @suggest_location
+    end
+
+    def rank
+      #@lock_rank.synchronize do
+        if @rank.nil?
+          if subsequents.nil? || subsequents.empty?
+            @rank = 0
+          else
+            max_rank = 0
+            subsequents.each do |subsq|
+              r = subsq.wrapper.rank
+              if max_rank < r
+                max_rank = r
+              end
+            end
+            if has_output_file?
+              step = 1
+            else
+              step = 0
+            end
+            @rank = max_rank + step
+          end
+          Log.debug "Task[#{name}] rank=#{@rank.inspect}"
+        end
+      #end
+      @rank
+    end
+
+    def file_size
+      @file_stat ? @file_stat.size : 0
+    end
+
+    def file_mtime
+      @file_stat ? @file_stat.mtime : Time.at(0)
+    end
+
+    def input_file_size
+      unless @input_file_size
+        @input_file_size = 0
+        prerequisites.each do |preq|
+          @input_file_size += Rake.application[preq].wrapper.file_size
+        end
+      end
+      @input_file_size
+    end
+
+    def input_file_mtime
+      if has_input_file? && @input_file_mtime.nil?
+        hash = Hash.new
+        max_sz = 0
+        prerequisites.each do |preq|
+          t = Rake.application[preq].wrapper
+          sz = t.file_size
+          if sz > 0
+            hash[t] = sz
+            if sz > max_sz
+              max_sz = sz
+            end
+          end
+        end
+        half_max_sz = max_sz / 2
+        hash.each do |t,sz|
+          if sz > half_max_sz
+            time = t.file_mtime
+            if @input_file_mtime.nil? || @input_file_mtime < time
+              @input_file_mtime = time
+            end
+          end
+        end
+      end
+      @input_file_mtime
+    end
+
+    def priority
+      if has_input_file? && @priority.nil?
+        sum_tm = 0
+        sum_sz = 0
+        prerequisites.each do |preq|
+          pq = Rake.application[preq].wrapper
+          sz = pq.file_size
+          if sz > 0
+            tm = pq.file_mtime - START_TIME
+            sum_tm += tm * sz
+            sum_sz += sz
+          end
+        end
+        if sum_sz > 0
+          @priority = sum_tm / sum_sz
+        else
+          @priority = 0
+        end
+        Log.debug "task_name=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
+      end
+      @priority || 0
+    end
+
+    def n_used_cores(host_info=nil)
+      @n_used_cores ||= @property.n_used_cores(host_info)
+    end
+
+  end
+end