pwrake 0.9.3

data/lib/pwrake/task_algorithm.rb

@@ -0,0 +1,171 @@
+module Pwrake
+
+  InvocationChain = Rake::InvocationChain
+  TaskArguments = Rake::TaskArguments
+
+  module TaskAlgorithm
+
+    def location
+      @location ||= []
+    end
+
+    def location=(a)
+      @location = a
+    end
+
+
+    def invoke_modify(*args)
+      task_args = TaskArguments.new(arg_names, args)
+      flag = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
+      start_time = Time.now
+      if flag
+        application.task_queue.enq_synchronize do
+          search_with_call_chain(self, task_args, InvocationChain::EMPTY)
+        end
+      else
+        search_with_call_chain(self, task_args, InvocationChain::EMPTY)
+      end
+      Log.info "-- search_tasks %.6fs" % (Time.now-start_time)
+
+      if conn = Pwrake.current_shell
+        @waiting_thread = nil
+        application.thread_loop(conn,self)
+      else
+        @waiting_thread = Thread.current
+        Log.debug "--- sleep in invoke_modify #{self.name} #{@waiting_thread.inspect}"
+        sleep
+        Log.debug "--- awake in invoke_modify"
+        pw_invoke
+      end
+    end
+
+
+    def pw_invoke
+      shell = Pwrake.current_shell
+      if shell
+        shell.current_task = self
+        log_host(shell.host)
+      end
+      @lock.synchronize do
+        return if @already_invoked
+        @already_invoked = true
+      end
+      pw_execute(@arg_data) if needed?
+      start_time = Time.now
+      pw_enq_subsequents
+      Log.info "-- pw_enq_subsequents(%s) %.6fs"%[self.name,Time.now-start_time]
+      shell.current_task = nil if shell
+    end
+
+
+    # Execute the actions associated with this task.
+    def pw_execute(args=nil)
+      args ||= Rake::EMPTY_TASK_ARGS
+      if application.options.dryrun
+        Log.info "** Execute (dry run) #{name}"
+        return
+      end
+      if application.options.trace
+        Log.info "** Execute #{name}"
+      end
+      application.enhance_with_matching_rule(name) if @actions.empty?
+      @actions.each do |act|
+        case act.arity
+        when 1
+          act.call(self)
+        else
+          act.call(self, args)
+        end
+      end
+    end
+
+    def pw_enq_subsequents
+      @lock.synchronize do
+        my_name = self.name
+        application.task_queue.enq_synchronize do
+          @subsequents.each do |t|        # <<--- competition !!!
+            t && t.check_and_enq(my_name)
+          end
+          @already_finished = true        # <<--- competition !!!
+        end
+      end
+    end
+
+    def check_and_enq(preq_name=nil)
+      @unfinished_prereq.delete(preq_name)
+      if @unfinished_prereq.empty?
+        if @waiting_thread
+          Log.debug "--- @waiting_thread.wakeup name=#{self.name} "
+          @waiting_thread.wakeup
+        else
+          Log.debug "--- check_and_enq enq name=#{self.name} "
+          application.task_queue.enq(self)
+        end
+      end
+    end
+
+    # Same as search, but explicitly pass a call chain to detect
+    # circular dependencies.
+    def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
+      new_chain = InvocationChain.append(self, invocation_chain)
+      @lock.synchronize do
+        if application.options.trace
+          Log.info "** Search #{name} #{format_search_flags}"
+        end
+
+        return true if @already_finished # <<--- competition !!!
+        @subsequents ||= []
+        @subsequents << subseq           # <<--- competition !!!
+
+        if ! @already_searched
+          @already_searched = true
+          @arg_data = task_args
+          search_prerequisites(task_args, new_chain)
+        end
+        return false
+      end
+    rescue Exception => ex
+      add_chain_to(ex, new_chain)
+      raise ex
+    end
+
+    # Search all the prerequisites of a task.
+    def search_prerequisites(task_args, invocation_chain) # :nodoc:
+      @unfinished_prereq = @prerequisites.dup
+      prerequisite_tasks.each { |prereq|
+        #prereq_args = task_args.new_scope(prereq.arg_names) # in vain
+        if prereq.search_with_call_chain(self, task_args, invocation_chain)
+          @unfinished_prereq.delete(prereq.name)
+        end
+      }
+      check_and_enq
+    end
+
+    # Format the trace flags for display.
+    def format_search_flags
+      flags = []
+      flags << "finished" if @already_finished
+      flags << "first_time" unless @already_searched
+      flags << "not_needed" unless needed?
+      flags.empty? ? "" : "(" + flags.join(", ") + ")"
+    end
+    private :format_search_flags
+
+
+    def log_host(exec_host)
+      # exec_host = Pwrake.current_shell.host
+      prereq_name = @prerequisites[0]
+      if kind_of?(Rake::FileTask) and prereq_name
+        Pwrake.application.count( @location, exec_host )
+        if @location and @location.include? exec_host
+          compare = "=="
+        else
+          compare = "!="
+        end
+        Log.info "-- access to #{prereq_name}: file_host=#{@location.inspect} #{compare} exec_host=#{exec_host}"
+      end
+    end
+
+  end
+
+end # module Pwrake

data/lib/pwrake/task_queue.rb

@@ -0,0 +1,167 @@
+module Pwrake
+
+  class TaskQueue
+
+    def initialize(*args)
+      @finished = false
+      @halt = false
+      @mutex = Mutex.new
+      @cv = ConditionVariable.new
+      @th_end = []
+      @enable_steal = true
+      @q = []
+      @reservation = {}
+      @reserved_q = {}
+    end
+
+    attr_reader :mutex
+    attr_accessor :enable_steal
+
+    def reserve(item)
+      @reservation[item] = Thread.current
+    end
+
+    def halt
+      @mutex.synchronize do
+        @halt = true
+      end
+    end
+
+    def resume
+      @mutex.synchronize do
+        @halt = false
+        @cv.broadcast
+      end
+    end
+
+    def enq_synchronize
+      if @halt
+        yield
+      else
+        @mutex.synchronize do
+          yield
+          enq_finish
+        end
+        @cv.broadcast
+      end
+      @reserved_q.keys.each do |th|
+        Log.debug "--- run #{th}";
+        th.run
+      end
+    end
+
+    def enq_finish
+    end
+
+
+    def enq(item,hint=nil)
+      if th = @reservation[item]
+        @reserved_q[th] = item
+      else
+        enq_impl(item,hint)
+      end
+    end
+
+
+    def enq_bak(item,hint=nil)
+      # Log.debug "--- #{self.class}#enq #{item.inspect}"
+      th = nil
+      if @halt
+        if th = @reservation[item]
+          @reserved_q[th] = item
+        else
+          enq_impl(item,hint)
+        end
+      else
+        @mutex.synchronize do
+          if th = @reservation[item]
+            @reserved_q[th] = item
+          else
+            enq_impl(item,hint)
+            @cv.signal
+          end
+        end
+      end
+      @reserved_q.keys.each{|th|
+        Log.debug "--- run #{th}";
+        th.run
+      }
+    end
+
+    def enq_impl(item,hint)
+      @q.push(item)          # FIFO Queue
+    end
+
+
+    def deq(hint=nil)
+      # Log.debug "--- #{self.class}#deq #{self.inspect}"
+      n = 0
+      loop do
+        @mutex.synchronize do
+          if @th_end.first == Thread.current
+            @th_end.shift
+            return false
+
+          elsif @halt
+            Log.debug "--- halt in #{self.class}#deq @q=#{@q.inspect}"
+            @cv.wait(@mutex)
+            n = 0
+
+          elsif item = @reserved_q.delete(Thread.current)
+            Log.debug "--- deq from reserved_q=#{item.inspect}"
+            return item
+
+          elsif empty? # no item in queue
+            Log.debug "--- empty=true in #{self.class}#deq @finished=#{@finished.inspect}"
+            if @finished
+              @cv.signal
+              return false
+            end
+            Log.debug "--- waiting in #{self.class}#deq @finished=#{@finished.inspect}"
+            @cv.wait(@mutex)
+            n = 0
+
+          else
+            if t = deq_impl(hint,n)
+              Log.debug "--- #{self.class}#deq #{t.inspect}"
+              return t
+            end
+            n += 1
+          end
+        end
+      end
+    end
+
+    def deq_impl(hint,n)
+      @q.shift               # FIFO Queue
+    end
+
+    def clear
+      @q.clear
+      @reserved_q.clear
+    end
+
+    def empty?
+      @q.empty? && @reserved_q.empty?
+    end
+
+    def finish
+      Log.debug "--- #{self.class}#finish"
+      @finished = true
+      @cv.signal
+    end
+
+    def stop
+      @mutex.synchronize do
+        clear
+        finish
+      end
+    end
+
+    def thread_end(th)
+      @th_end.push(th)
+      @cv.broadcast
+    end
+  end
+
+end

data/lib/pwrake/timer.rb

@@ -0,0 +1,22 @@
+module Pwrake
+
+  class Timer
+
+    def initialize(prefix,*extra)
+      @prefix = prefix
+      @start_time = Time.now
+      str = "%s[start]:%s %s" %
+        [@prefix, Log.fmt_time(@start_time), extra.join(' ')]
+      Log.info(str)
+    end
+
+    def finish(*extra)
+      end_time = Time.now
+      elap_time = end_time - @start_time
+      str = "%s[end]:%s elap=%.3f %s" %
+        [@prefix, Log.fmt_time(end_time), elap_time, extra.join(' ')]
+      Log.info(str)
+    end
+
+  end
+end

data/lib/pwrake/version.rb

@@ -0,0 +1,3 @@
+module Pwrake
+  VERSION = "0.9.3"
+end

data/lib/pwrake.rb

@@ -0,0 +1,18 @@
+require "thread"
+require "pathname"
+
+require "pwrake/version"
+
+require "pwrake/logger"
+require "pwrake/timer"
+require "pwrake/counter"
+require "pwrake/file_utils"
+require "pwrake/profiler"
+require "pwrake/shell"
+require "pwrake/task_algorithm"
+require "pwrake/task_queue"
+require "pwrake/option"
+require "pwrake/master"
+require "pwrake/application"
+
+require "pwrake/rake_modify"

data/pwrake.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pwrake/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "pwrake"
+  gem.version       = Pwrake::VERSION
+  gem.authors       = ["Masahiro TANAKA"]
+  gem.email         = ["masa16.tanaka@gmail.com"]
+  gem.description   = %q{Parallel workflow extension for Rake}
+  gem.summary       = %q{Adding Parallel and Distributed feature to Rake}
+  gem.homepage      = "http://masa16.github.com/pwrake"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end