autobuild 1.6.0.b2 → 1.6.0.b3

data/Manifest.txt

@@ -21,9 +21,11 @@ lib/autobuild/packages/autotools.rb
 lib/autobuild/packages/cmake.rb
 lib/autobuild/packages/dummy.rb
 lib/autobuild/packages/genom.rb
+lib/autobuild/packages/gnumake.rb
 lib/autobuild/packages/import.rb
 lib/autobuild/packages/orogen.rb
 lib/autobuild/packages/pkgconfig.rb
+lib/autobuild/parallel.rb
 lib/autobuild/pkgconfig.rb
 lib/autobuild/reporting.rb
 lib/autobuild/subcommand.rb

data/lib/autobuild/packages/gnumake.rb

@@ -0,0 +1,48 @@
+module Autobuild
+    def self.make_is_gnumake?(path = Autobuild.tool(:make))
+        @make_is_gnumake ||= Hash.new
+        if @make_is_gnumake.has_key?(path)
+            @make_is_gnumake[path]
+        else
+            result = `#{path} --version`
+            @make_is_gnumake[path] = $?.success? &&
+                (result.split("\n").first =~ /GNU Make/)
+        end
+    end
+
+    def self.make_has_j_option?(path = Autobuild.tool(:make))
+        make_is_gnumake?(path)
+    end
+
+    def self.make_has_gnumake_jobserver?(path = Autobuild.tool(:make))
+        make_is_gnumake?(path)
+    end
+
+    def self.make_subcommand(pkg, phase, *options, &block)
+        reserved = nil
+        cmd_path = Autobuild.tool(:make)
+        cmd = [cmd_path]
+        if make_has_j_option?(cmd_path) && pkg.parallel_build_level != 1
+            if manager = Autobuild.parallel_task_manager
+                job_server = manager.job_server
+                if !make_has_gnumake_jobserver?(cmd_path) || (pkg.parallel_build_level != Autobuild.parallel_build_level)
+                    reserved = pkg.parallel_build_level
+                    job_server.get(reserved - 1) # We already have one token taken by autobuild itself
+                    cmd << "-j#{pkg.parallel_build_level}"
+                else
+                    cmd << "--jobserver-fds=#{job_server.rio.fileno},#{job_server.wio.fileno}" << "-j"
+                end
+            else
+                cmd << "-j#{pkg.parallel_build_level}"
+            end
+        end
+
+        cmd.concat(options)
+        Subprocess.run(pkg, phase, *cmd, &block)
+
+    ensure
+        if reserved
+            job_server.put(reserved)
+        end
+    end
+end

data/lib/autobuild/parallel.rb

@@ -0,0 +1,216 @@
+require 'thread'
+
+module Autobuild
+    # This is a rewrite of the Rake task invocation code to use parallelism
+    #
+    # Since autobuild does not use task arguments, we don't support them for
+    # simplicity
+    class RakeTaskParallelism
+        attr_reader :available_workers
+        attr_reader :finished_workers
+        attr_reader :workers
+
+        attr_reader :job_server
+
+        attr_reader :roots
+        attr_reader :tasks
+        attr_reader :reverse_dependencies
+
+        class JobServer
+            attr_reader :rio
+            attr_reader :wio
+
+            def initialize(level)
+                @rio, @wio = IO.pipe
+                put(level)
+            end
+            def get(token_count = 1)
+                @rio.read(token_count)
+            end
+            def put(token_count = 1)
+                @wio.write(" " * token_count)
+            end
+        end
+
+        def initialize(level = Autobuild.parallel_build_level)
+            @job_server = JobServer.new(level)
+            @available_workers = Array.new
+            @finished_workers = Queue.new
+            @workers = Array.new
+
+        end
+
+        def process_finished_task(task, processed, queue)
+            processed << task
+            #puts "finished #{task} (#{task.object_id}), considering dependencies"
+            reverse_dependencies[task].each do |candidate|
+                if candidate.prerequisite_tasks.all? { |t| processed.include?(t) }
+                    #puts "  adding #{candidate}"
+                    queue << candidate
+                else
+                    #puts "  rejecting #{candidate} (#{candidate.prerequisite_tasks.find_all { |t| !processed.include?(t) }.map(&:name).join(", ")})"
+                end
+            end
+        end
+
+        def wait_for_worker_to_end(processed, queue)
+            w = finished_workers.pop
+            finished_task, error = w.last_result
+            available_workers << w
+            if error
+                if available_workers.size != workers.size
+                    Autobuild.message "got an error doing parallel processing, waiting for pending jobs to end"
+                end
+                finish_pending_work
+                raise error
+            end
+
+            process_finished_task(finished_task, processed, queue)
+        end
+
+        def discover_dependencies(t)
+            return if tasks.include?(t) # already discovered or being discovered
+            #puts "adding #{t}"
+            tasks << t
+
+            t.prerequisite_tasks.each do |dep_t|
+                reverse_dependencies[dep_t] << t
+                discover_dependencies(dep_t)
+            end
+        end
+
+        # Invokes the provided tasks. Unlike the rake code, this is a toplevel
+        # algorithm that does not use recursion
+        def invoke_parallel(required_tasks)
+            @tasks = Set.new
+            @reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }
+            required_tasks.each do |t|
+                discover_dependencies(t)
+            end
+            @roots = tasks.find_all { |t| t.prerequisite_tasks.empty? }.to_set
+
+            #puts "roots:"
+            roots.each do |t|
+                #puts "  #{t}"
+            end
+            #puts
+            
+            # Build a reverse dependency graph (i.e. a mapping from a task to
+            # the tasks that depend on it)
+
+            # This is kind-of a topological sort. However, we don't do the full
+            # topological sort since we would then have to scan all tasks each
+            # time for tasks that have no currently running prerequisites
+
+            # The queue is the set of tasks for which all prerequisites have
+            # been successfully executed (or where not needed). I.e. it is the
+            # set of tasks that can be queued for execution.
+            queue = roots.to_a
+            processed = Set.new
+            while true
+                if queue.empty?
+                    # If we have pending workers, wait for one to be finished
+                    # until either they are all finished or the queue is not
+                    # empty anymore
+                    while queue.empty? && available_workers.size != workers.size
+                        wait_for_worker_to_end(processed, queue)
+                    end
+
+                    if queue.empty? && available_workers.size == workers.size
+                        break
+                    end
+                end
+
+                pending_task = queue.pop
+                #puts "#{processed.size} tasks processed so far, #{tasks.size} total"
+                if pending_task.instance_variable_get(:@already_invoked) || !pending_task.needed?
+                    process_finished_task(pending_task, processed, queue)
+                    next
+                end
+
+                # Get a job server token
+                job_server.get
+
+                while !finished_workers.empty?
+                    wait_for_worker_to_end(processed, queue)
+                end
+
+                # We do have a job server token, so we are allowed to allocate a
+                # new worker if none are available
+                if available_workers.empty?
+                    w = Worker.new(job_server, finished_workers)
+                    available_workers << w
+                    workers << w
+                end
+
+                worker = available_workers.pop
+                #puts "queueing #{pending_task}"
+                worker.queue(pending_task)
+            end
+
+            if processed.size != tasks.size
+                with_cycle = tasks.to_set
+                (tasks.to_set - processed).each do |pending_task|
+                    pending_task.prerequisite_tasks.each do |t|
+                        if !processed.include?(t)
+                            #puts "#{pending_task} => #{t}"
+                        end
+                    end
+                end
+                raise "cycle in task graph"
+                raise "cycle in task graph: #{with_cycle.map(&:name).sort.join(", ")}"
+            end
+        end
+
+        class Worker
+            attr_reader :job_server
+
+            def initialize(job_server, finished_workers)
+                @job_server = job_server
+                @finished_workers = finished_workers
+                @input = Queue.new
+                @thread = Thread.new do
+                    loop do
+                        task = @input.pop
+                        do_task(task)
+                    end
+                end
+            end
+
+            def do_task(task)
+                @last_error = nil
+                task_args = Rake::TaskArguments.new(task.arg_names, [])
+                task.instance_variable_set(:@already_invoked, true)
+                task.send(:execute, task_args)
+                @last_finished_task = task
+            rescue ::Exception => e
+                @last_finished_task = task
+                @last_error = e
+            ensure
+                job_server.put
+                @finished_workers.push(self)
+            end
+
+            def last_result
+                return @last_finished_task, @last_error
+            end
+
+            def queue(task)
+                @input.push(task)
+            end
+        end
+
+        def finish_pending_work
+            while available_workers.size != workers.size
+                w = finished_workers.pop
+                available_workers << w
+            end
+        end
+    end
+
+    class << self
+        attr_accessor :parallel_task_manager
+    end
+end
+
+

data/lib/autobuild/reporting.rb

@@ -16,15 +16,15 @@ require 'autobuild/exceptions'
 
 module Autobuild
     def self.message(*args)
-        if !progress_messages.empty?
+        if @last_progress_msg
             puts
+            @last_progress_msg = nil
         end
         if args.empty?
             puts
         else
             puts "#{color(*args)}"
         end
-        @last_msg = nil
     end
 
     class << self
@@ -85,11 +85,11 @@ module Autobuild
 
     def self.display_progress
         msg = "#{progress_messages.map(&:last).join(" | ")}"
-        if @last_msg && @last_msg.length > msg.length
-            print "\r" + " " * @last_msg.length
+        if @last_progress_msg && @last_progress_msg.length > msg.length
+            print "\r" + " " * @last_progress_msg.length
         end
         print "\r  #{msg}"
-        @last_msg = msg
+        @last_progress_msg = msg
     end
 
     # The exception type that is used to report multiple errors that occured

data/lib/autobuild/version.rb

@@ -1,5 +1,5 @@
 module Autobuild
-    VERSION = "1.6.0.b2" unless defined? Autobuild::VERSION
+    VERSION = "1.6.0.b3" unless defined? Autobuild::VERSION
 end
 
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: autobuild
 version: !ruby/object:Gem::Version 
-  hash: 51
+  hash: 49
   prerelease: 6
   segments: 
   - 1
   - 6
   - 0
   - b
-  - 2
-  version: 1.6.0.b2
+  - 3
+  version: 1.6.0.b3
 platform: ruby
 authors: 
 - Sylvain Joyeux
@@ -17,7 +17,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-08 00:00:00 Z
+date: 2012-03-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rake
@@ -57,13 +57,14 @@ dependencies:
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 47
         segments: 
         - 2
-        - 12
-        version: "2.12"
+        - 8
+        - 0
+        version: 2.8.0
   type: :development
   version_requirements: *id003
 description: |-
@@ -118,9 +119,11 @@ files:
 - lib/autobuild/packages/cmake.rb
 - lib/autobuild/packages/dummy.rb
 - lib/autobuild/packages/genom.rb
+- lib/autobuild/packages/gnumake.rb
 - lib/autobuild/packages/import.rb
 - lib/autobuild/packages/orogen.rb
 - lib/autobuild/packages/pkgconfig.rb
+- lib/autobuild/parallel.rb
 - lib/autobuild/pkgconfig.rb
 - lib/autobuild/reporting.rb
 - lib/autobuild/subcommand.rb
@@ -135,7 +138,6 @@ files:
 - test/test_import_tar.rb
 - test/test_subcommand.rb
 - test/tools.rb
-- .gemtest
 homepage: 
 licenses: []