drake 0.8.4.1.1.0 → 0.8.4.1.2.0

data/lib/rake/parallel.rb

@@ -1,43 +1,148 @@
+#
+# Parallel task execution for Rake.
+#
+# The comp_tree package is used to create a computation tree which
+# executes Rake tasks.
+#
+# Tasks are first collected with a single-threaded dry run.  This
+# expands file rules, resolves prequisite names and finds task
+# arguments.  A computation tree is then built with the gathered
+# tasks.
+#
+# Note that prerequisites are context-dependent; it is therefore not
+# possible to create a 1-to-1 mapping between Rake::Tasks and
+# CompTree::Nodes.
+#
+# Author: James M. Lawrence <quixoticsycophant@gmail.com>
+#
 
-require 'rake/comp_tree/comp_tree'
+require 'comp_tree'
 
 module Rake
-  module TaskManager
-    def invoke_parallel(root_task_name) # :nodoc:
-      CompTree.build do |driver|
-        #
-        # Build the computation tree from task prereqs.
-        #
-        self.parallel.tasks.each_pair { |task_name, cache|
-          task = self[task_name]
-          task_args, prereqs = cache
-          children_names = prereqs.map { |child| child.name }
-          driver.define(task_name, *children_names) {
-            task.execute(task_args)
-          }
-        }
+  module Parallel  #:nodoc:
+    class Driver
+      # Tasks collected during the dry-run phase.
+      attr_reader :tasks
+
+      # Prevent invoke inside invoke.
+      attr_reader :mutex
+
+      def initialize
+        @tasks = Hash.new
+        @mutex = Mutex.new
+      end
 
-        root_node = driver.nodes[root_task_name]
-
-        #
-        # If there were nothing to do, there would be no root node.
-        #
-        if root_node
-          #
-          # Mark computation nodes without a function as computed.
-          #
-          root_node.each_downward { |node|
-            unless node.function
-              node.computed = true
+      #
+      # Top-level parallel invocation.
+      #
+      # Called from Task#invoke (routed through Task#invoke_parallel).
+      #
+      def invoke(threads, task, *task_args)
+        if @mutex.try_lock
+          begin
+            @tasks.clear
+
+            # dry run task collector
+            task.invoke_serial(*task_args)
+
+            if @tasks.has_key? task
+              # hand it off to comp_tree
+              compute(task, threads)
             end
-          }
+          ensure
+            @mutex.unlock
+          end
+        else
+          raise InvokeInsideInvoke
+        end
+      end
+
+      #
+      # Build and run the computation tree.
+      #
+      # Called from Parallel::Driver#invoke.
+      #
+      def compute(root_task, threads)
+        CompTree.build do |driver|
+          # keep this around for optimization
+          needed_prereq_names = Array.new
+
+          @tasks.each_pair do |task, (task_args, prereqs)|
+            # if a prereq is not needed then it didn't get into @tasks
+            needed_prereq_names.clear
+            prereqs.each do |prereq|
+              needed_prereq_names << prereq.name if @tasks.has_key? prereq
+            end
+
+            # define a computation node which executes the task
+            driver.define(task.name, *needed_prereq_names) {
+              task.execute(task_args)
+            }
+          end
 
-          #
-          # Launch the computation.
-          #
-          driver.compute(root_node.name, self.num_threads)
+          # punch it
+          driver.compute(root_task.name, threads)
         end
       end
     end
+  
+    module ApplicationMixin
+      def parallel
+        @parallel ||= Driver.new
+      end
+    end
+
+    module TaskMixin
+      #
+      # Top-level parallel invocation.
+      #
+      # Called from Task#invoke.
+      #
+      def invoke_parallel(*task_args)
+        application.parallel.invoke(application.options.threads, self, *task_args)
+      end
+
+      #
+      # Collect tasks for parallel execution.
+      #
+      # Called from Task#invoke_with_call_chain.
+      #
+      def invoke_with_call_chain_collector(task_args, new_chain, previous_chain)
+        prereqs = invoke_prerequisites_collector(task_args, new_chain)
+        parallel = application.parallel
+        if needed? or parallel.tasks[self]
+          parallel.tasks[self] = [task_args, prereqs]
+          unless previous_chain == InvocationChain::EMPTY
+            #
+            # Touch the parent to propagate 'needed?' upwards.  This
+            # works because the recursion is depth-first.
+            #
+            parallel.tasks[previous_chain.value] = true
+          end
+        end
+      end
+      
+      #
+      # Dry-run invoke prereqs and return the prereq instances.
+      # This also serves to avoid MultiTask#invoke_prerequisites.
+      #
+      # Called from Task#invoke_with_call_chain_collector.
+      #
+      def invoke_prerequisites_collector(task_args, invocation_chain)
+        @prerequisites.map { |n|
+          invoke_prerequisite(n, task_args, invocation_chain)
+        }
+      end
+    end
+  end
+
+  #
+  # Error indicating Task#invoke was called inside Task#invoke
+  # during parallel execution.
+  #
+  class InvokeInsideInvoke < StandardError
+    def message
+      "Cannot call Task#invoke within a task during parallel execution."
+    end
   end
 end

data/test/filecreation.rb

@@ -1,7 +1,9 @@
 #!/usr/bin/env ruby
 
 module FileCreation
+  ANCIENT_FILE = 'testdata/anc'
   OLDFILE = "testdata/old"
+  MIDDLE_AGED_FILE = "testdata/mid"
   NEWFILE = "testdata/new"
 
   def create_timed_files(oldfile, *newfiles)
@@ -15,6 +17,16 @@ module FileCreation
     end
   end
 
+  def create_dispersed_timed_files(*files)
+    create_file(files.first)
+    (1...files.size).each do |index|
+      while create_file(files[index]) <= File.stat(files[index - 1]).mtime
+        sleep(0.1)
+        File.delete(files[index])
+      end
+    end
+  end
+
   def create_dir(dirname)
     FileUtils.mkdir_p(dirname) unless File.exist?(dirname)
     File.stat(dirname).mtime

data/test/parallel_setup.rb

@@ -0,0 +1,3 @@
+require 'rake'
+Rake.application.options.threads = 8
+puts "- Testing parallel execution"

data/test/rake_test_setup.rb

@@ -7,6 +7,7 @@ rescue LoadError
   # got no gems
 end
 
+require 'thread'
 require 'flexmock/test_unit'
 
 if RUBY_VERSION >= "1.9.0"
@@ -22,3 +23,23 @@ module TestMethods
     assert_raise(ex, msg, &block)
   end
 end
+
+class SerializedArray
+  def initialize
+    @mutex = Mutex.new
+    @array = Array.new
+  end
+
+  Array.public_instance_methods.each do |method_name|
+    unless method_name =~ %r!\A__! or method_name =~ %r!\A(object_)?id\Z!
+      # TODO: jettison 1.8.6; use define_method with |&block|
+      eval %{
+        def #{method_name}(*args, &block)
+          @mutex.synchronize {
+            @array.send('#{method_name}', *args, &block)
+          }
+        end
+      }
+    end
+  end
+end

data/test/serial_setup.rb

@@ -0,0 +1,3 @@
+require 'rake'
+Rake.application.options.threads = 1
+puts "- Testing serial execution"

data/test/test_application.rb

@@ -22,6 +22,7 @@ class TestApplication < Test::Unit::TestCase
 
   def setup
     @app = Rake::Application.new
+    @app.options.threads = Rake.application.options.threads
     @app.options.rakelib = []
   end
 
@@ -570,6 +571,7 @@ class TestApplicationOptions < Test::Unit::TestCase
   def command_line(*options)
     options.each do |opt| ARGV << opt end
     @app = Rake::Application.new
+    @app.options.threads = Rake.application.options.threads
     def @app.exit(*args)
       throw :system_exit, :exit
     end
@@ -585,6 +587,7 @@ end
 class TestTaskArgumentParsing < Test::Unit::TestCase
   def setup
     @app = Rake::Application.new
+    @app.options.threads = Rake.application.options.threads
   end
   
   def test_name_only
@@ -630,6 +633,7 @@ class TestTaskArgumentParsing < Test::Unit::TestCase
 
   def test_terminal_width_using_env
     app = Rake::Application.new
+    app.options.threads = Rake.application.options.threads
     in_environment('RAKE_COLUMNS' => '1234') do
       assert_equal 1234, app.terminal_width
     end
@@ -637,6 +641,7 @@ class TestTaskArgumentParsing < Test::Unit::TestCase
 
   def test_terminal_width_using_stty
     app = Rake::Application.new
+    app.options.threads = Rake.application.options.threads
     flexmock(app,
       :unix? => true,
       :dynamic_width_stty => 1235,
@@ -648,6 +653,7 @@ class TestTaskArgumentParsing < Test::Unit::TestCase
 
   def test_terminal_width_using_tput
     app = Rake::Application.new
+    app.options.threads = Rake.application.options.threads
     flexmock(app,
       :unix? => true,
       :dynamic_width_stty => 0,
@@ -659,6 +665,7 @@ class TestTaskArgumentParsing < Test::Unit::TestCase
 
   def test_terminal_width_using_hardcoded_80
     app = Rake::Application.new
+    app.options.threads = Rake.application.options.threads
     flexmock(app, :unix? => false)
     in_environment('RAKE_COLUMNS' => nil) do
       assert_equal 80, app.terminal_width
@@ -667,6 +674,7 @@ class TestTaskArgumentParsing < Test::Unit::TestCase
 
   def test_terminal_width_with_failure
     app = Rake::Application.new
+    app.options.threads = Rake.application.options.threads
     flexmock(app).should_receive(:unix?).and_throw(RuntimeError)
     in_environment('RAKE_COLUMNS' => nil) do
       assert_equal 80, app.terminal_width

data/test/test_definitions.rb

@@ -48,14 +48,18 @@ class TestDefinitions < Test::Unit::TestCase
   end
 
   def test_incremental_definitions
-    runs = []
+    runs = SerializedArray.new
     task :t1 => [:t2] do runs << "A"; 4321 end
     task :t1 => [:t3] do runs << "B"; 1234 end
     task :t1 => [:t3]
     task :t2
     task :t3
     Task[:t1].invoke
-    assert_equal ["A", "B"], runs
+    if Rake.application.options.threads == 1
+      assert_equal ["A", "B"], runs
+    else
+      assert_equal ["A", "B"], runs.sort
+    end
     assert_equal ["t2", "t3"], Task[:t1].prerequisites
   end
 

data/test/test_file_task.rb

@@ -12,11 +12,17 @@ class TestFileTask < Test::Unit::TestCase
   include FileCreation
   include TestMethods
 
+  FILES = [ANCIENT_FILE, OLDFILE, MIDDLE_AGED_FILE, NEWFILE] 
+
   def setup
     Task.clear
-    @runs = Array.new
-    FileUtils.rm_f NEWFILE
-    FileUtils.rm_f OLDFILE
+    @runs = SerializedArray.new
+    FileUtils.rm_f FILES
+  end
+
+  def test_create_dispersed_timed_files
+    create_dispersed_timed_files(*FILES)
+    assert_equal FILES, FILES.sort_by { |f| File.stat(f).mtime }
   end
 
   def test_file_need
@@ -73,6 +79,46 @@ class TestFileTask < Test::Unit::TestCase
     assert_nothing_raised do Task[OLDFILE].invoke end
   end
 
+  def test_old_file_in_between
+    create_dispersed_timed_files(*FILES)
+
+    file MIDDLE_AGED_FILE => OLDFILE do |t|
+      @runs << t.name
+    end
+    file OLDFILE => NEWFILE do |t|
+      @runs << t.name
+      touch OLDFILE, :verbose => false
+    end
+    file NEWFILE do |t|
+      @runs << t.name
+    end
+
+    Task[MIDDLE_AGED_FILE].invoke
+    assert_equal([OLDFILE, MIDDLE_AGED_FILE], @runs)
+  end
+
+  def test_two_old_files_in_between
+    create_dispersed_timed_files(*FILES)
+
+    file MIDDLE_AGED_FILE => OLDFILE do |t|
+      @runs << t.name
+    end
+    file OLDFILE => ANCIENT_FILE do |t|
+      @runs << t.name
+      touch OLDFILE, :verbose => false
+    end
+    file ANCIENT_FILE => NEWFILE do |t|
+      @runs << t.name
+      touch ANCIENT_FILE, :verbose => false
+    end
+    file NEWFILE do |t|
+      @runs << t.name
+    end
+
+    Task[MIDDLE_AGED_FILE].invoke
+    assert_equal([ANCIENT_FILE, OLDFILE, MIDDLE_AGED_FILE], @runs)
+  end
+
   # I have currently disabled this test.  I'm not convinced that
   # deleting the file target on failure is always the proper thing to
   # do.  I'm willing to hear input on this topic.

data/test/test_multitask.rb

@@ -9,7 +9,7 @@ class TestMultiTask < Test::Unit::TestCase
 
   def setup
     Task.clear
-    @runs = Array.new
+    @runs = SerializedArray.new
   end
 
   def test_running_multitasks

data/test/test_parallel.rb

@@ -1,56 +1,195 @@
 
 require 'rbconfig'
 require 'test/unit'
+require 'rake'
+require 'test/rake_test_setup'
 
-PARALLEL_TEST_MESSAGE = <<'EOS'
+if Rake.application.options.threads > 1
+  class TestParallel < Test::Unit::TestCase
+    VISUALS = false
+    TIME_STEP = 0.25
+    TIME_EPSILON = 0.05
+    MAX_THREADS = 5
 
+    def trace(str)
+      if VISUALS
+        puts str
+      end
+    end
 
-Task graph for sample parallel execution:
-
-                default
-                  / \
-                 /   \
-                a     b
-               / \
-              /   \
-             x     y
+    def assert_order(expected, actual)
+      assert_in_delta(expected*TIME_STEP, actual, TIME_EPSILON)
+    end
 
-EOS
-      
-if Rake.application.num_threads > 1
-  class TestSimpleParallel < Test::Unit::TestCase
-    def setup
-      puts PARALLEL_TEST_MESSAGE
+    def teardown
+      Rake::Task.clear
     end
 
     def test_parallel
-      here = File.dirname(__FILE__)
-      rake = File.expand_path("#{here}/../bin/rake")
-
-      ENV["RUBYLIB"] = lambda {
-        lib = File.expand_path("#{here}/../lib")
-        current = ENV["RUBYLIB"]
-        sep = Rake.application.windows? ? ";" : ":"
-        if current
-          "#{lib}#{sep}#{current}"
+      trace GRAPH
+
+      data = Hash.new { |hash, key| hash[key] = Hash.new }
+      
+      (1..MAX_THREADS).each { |threads|
+        app = Rake::Application.new
+        app.options.threads = threads
+
+        app.define_task Rake::Task, :default => [:a, :b]
+        app.define_task Rake::Task, :a => [:x, :y]
+        app.define_task Rake::Task, :b
+
+        mutex = Mutex.new
+        STDOUT.sync = true
+        start_time = nil
+
+        %w[default a b x y].each { |task_name|
+          app.define_task Rake::Task, task_name.to_sym do
+            mutex.synchronize {
+              trace "task #{task_name}"
+              data[threads][task_name] = Time.now - start_time
+            }
+            sleep(TIME_STEP)
+          end
+        }
+      
+        trace "-"*50
+        trace "threads: #{threads}"
+        start_time = Time.now
+        app[:default].invoke
+      }
+
+      assert_order(0, data[1]["x"])
+      assert_order(1, data[1]["y"])
+      assert_order(2, data[1]["a"])
+      assert_order(3, data[1]["b"])
+      assert_order(4, data[1]["default"])
+
+      assert_order(0, data[2]["x"])
+      assert_order(0, data[2]["y"])
+      assert_order(1, data[2]["a"])
+      assert_order(1, data[2]["b"])
+      assert_order(2, data[2]["default"])
+
+      (3..MAX_THREADS).each { |threads|
+        assert_order(0, data[threads]["x"])
+        assert_order(0, data[threads]["y"])
+        assert_order(0, data[threads]["b"])
+        assert_order(1, data[threads]["a"])
+        assert_order(2, data[threads]["default"])
+      }
+    end 
+
+    def test_invoke_inside_invoke
+      assert_raises(Rake::InvokeInsideInvoke) {
+        app = Rake::Application.new
+        app.options.threads = 4
+        app.define_task Rake::Task, :root do
+          app[:stuff].invoke
+        end
+        app.define_task Rake::Task, :stuff do
+          flunk
+        end
+        app[:root].invoke
+      }
+    end
+    
+    def test_randomize
+      size = 100
+      [false, true].each do |randomize|
+        memo = SerializedArray.new
+        app = Rake::Application.new
+        app.define_task Rake::Task, :root
+        size.times { |n|
+          app.define_task Rake::Task, :root => n.to_s
+          app.define_task Rake::Task, n.to_s do
+            memo << n
+          end
+        }
+        app.options.randomize = randomize
+        app[:root].invoke
+        numbers = (0...size).to_a
+        if randomize
+          assert_not_equal(numbers, memo)
         else
-          lib
+          assert_equal(numbers, memo)
         end
-      }.call
-
-      [
-       ["Rakefile.simple", true],
-       ["Rakefile.seq", false],
-      ].each { |file, disp|
-        (1..5).each { |n|
-          args = [rake, "--threads", n.to_s, "-s", "-f", "test/#{file}"]
-          if disp
-            puts "\nvisual check: #{n} thread#{n > 1 ? 's' : ''}"
-            puts args.join(" ")
+      end
+    end
+
+    def test_multitask_not_called
+      # ensure MultiTask methods are not called by hijacking all of them
+
+      originals = [
+       :private_instance_methods,
+       :protected_instance_methods, 
+       :public_instance_methods,
+      ].inject(Hash.new) { |acc, query|
+        result = Rake::MultiTask.send(query, false).inject(Hash.new) {
+          |sub_acc, method_name|
+          sub_acc.merge!(
+            method_name => Rake::MultiTask.instance_method(method_name)
+          )
+        }
+        acc.merge!(result)
+      }
+
+      memo = SerializedArray.new
+
+      Rake::MultiTask.module_eval {
+        originals.each_pair { |method_name, method_object|
+          remove_method method_name
+          define_method method_name do |*args|
+            # missing |&block| due to 1.8.6, but not using it anyway
+            memo << 'called'
+            method_object.bind(self).call(*args)
           end
-          assert(ruby(*args))
         }
       }
+
+      begin
+        app = Rake::Application.new
+
+        define = lambda {
+          app.define_task Rake::Task, task(:x) { }
+          app.define_task Rake::Task, task(:y) { }
+          app.define_task Rake::MultiTask, :root => [:x, :y]
+        }
+
+        app.options.threads = 1
+        define.call
+        app[:root].invoke
+        assert_equal ['called'], memo
+
+        app.clear
+        memo.clear
+        assert_raises(RuntimeError) { app[:root].invoke }
+        
+        app.options.threads = 4
+        define.call
+        app[:root].invoke
+        assert_equal [], memo
+      ensure
+        Rake::MultiTask.module_eval {
+          originals.each_pair { |method_name, method_object|
+            remove_method method_name
+            define_method method_name, method_object
+          }
+        }
+      end
     end
+
+    GRAPH = <<-'EOS'
+
+    Task graph for sample parallel execution:
+  
+                  default
+                    / \
+                   /   \
+                  a     b
+                 / \
+                /   \
+               x     y
+  
+    EOS
   end
 end