autobuild 1.3.3 → 1.4.0

data/Changes.txt

@@ -1,3 +1,11 @@
+== Version 1.4.0
+* cmake package handler now displays progress value
+* added support for parallel builds
+* added the long-awaited --force-build and --rebuild options.
+  The first one forces the call to all the build steps, but without deleting the
+  build products. The second one deletes all build products and retriggers a
+  full build.
+
 == Version 1.3.3
 * fix: changing Orogen.corba now affects packages that have already been
   declared but for which the #corba attribute has not been set explicitely to

data/Manifest.txt

@@ -18,6 +18,7 @@ lib/autobuild/importer.rb
 lib/autobuild/package.rb
 lib/autobuild/packages/autotools.rb
 lib/autobuild/packages/cmake.rb
+lib/autobuild/packages/dummy.rb
 lib/autobuild/packages/genom.rb
 lib/autobuild/packages/import.rb
 lib/autobuild/packages/orogen.rb

data/Rakefile

@@ -16,3 +16,13 @@ Hoe.spec 'autobuild' do
         ['utilrb', '>= 1.3.3']
 end
 
+Rake.clear_tasks(/publish_docs/)
+task 'publish_docs' => 'doc' do
+    if !system('./update_github')
+        raise "cannot update the gh-pages branch for GitHub"
+    end
+    if !system('git', 'push', 'origin', 'gh-pages')
+        raise "cannot push the documentation"
+    end
+end
+

data/lib/autobuild.rb

@@ -1,5 +1,5 @@
 module Autobuild
-    VERSION = "1.3.3" unless defined? Autobuild::VERSION
+    VERSION = "1.4.0" unless defined? Autobuild::VERSION
 end
 
 require 'autobuild/config'
@@ -18,6 +18,7 @@ require 'autobuild/packages/genom'
 require 'autobuild/packages/import'
 require 'autobuild/packages/orogen'
 require 'autobuild/packages/pkgconfig'
+require 'autobuild/packages/dummy'
 require 'autobuild/pkgconfig'
 require 'autobuild/reporting'
 require 'autobuild/subcommand'

data/lib/autobuild/config.rb

@@ -18,6 +18,8 @@ end
 # debug:: more verbose than 'verbose': displays Rake's debugging output
 # do_update:: if we should update the packages
 # do_build:: if we should build the packages
+# do_forced_build:: if we should forcefully trigger all the packages build phases
+# do_rebuild:: if we should cleanly rebuild every packages
 # do_doc:: if we should produce the documentation
 # doc_errors:: if errors during the documentation generation are treated as errors
 # daemonize:: if the build should go into daemon mode (only if the daemons gem is available)
@@ -30,7 +32,7 @@ end
 module Autobuild
     class << self
         %w{ nice srcdir prefix
-            verbose debug do_update do_build only_doc do_doc doc_errors
+            verbose debug do_update do_build do_rebuild do_forced_build only_doc do_doc doc_errors
             daemonize clean_log packages default_packages
             doc_prefix keep_oldlogs}.each do |name|
             attr_accessor name
@@ -48,7 +50,7 @@ module Autobuild
     end
     DEFAULT_OPTIONS = { :nice => 0,
         :srcdir => Dir.pwd, :prefix => Dir.pwd, :logdir => nil,
-        :verbose => false, :debug => false, :do_build => true, :do_update => true, 
+        :verbose => false, :debug => false, :do_build => true, :do_forced_build => false, :do_rebuild => false, :do_update => true, 
         :daemonize => false, :packages => [], :default_packages => [],
         :only_doc => false, :do_doc => true, :doc_errors => false,
         :doc_prefix => 'doc', :keep_oldlogs => false }
@@ -157,6 +159,8 @@ module Autobuild
 		end
                 opts.on("--no-update", "update already checked-out sources") do |@do_update| end
                 opts.on("--no-build",  "only prepare packages, do not build them") do |@do_build| end 
+                opts.on("--forced-build",  "force the trigger of all the build commands") do |@do_forced_build| end 
+                opts.on("--rebuild",  "clean and rebuild") do |@do_forced_build| end 
                 opts.on("--only-doc", "only generate documentation") do |@only_doc| end
                 opts.on("--no-doc", "don't generate documentation") do |@do_doc| end
                 opts.on("--doc-errors", "treat documentation failure as error") do |@doc_errors| end
@@ -211,6 +215,19 @@ module Autobuild
             end
         end
 
+        if Autobuild.do_rebuild
+            packages.each do |pkg_name|
+                Autobuild::Package[pkg_name].prepare_for_rebuild
+            end
+            # And delete the prefix !
+            FileUtils.rm_rf Autobuild.prefix
+
+        elsif Autobuild.do_forced_build
+            packages.each do |pkg_name|
+                Autobuild::Package[pkg_name].prepare_for_forced_build
+            end
+        end
+
         if Autobuild.only_doc
             phases  = ['doc']
         else

data/lib/autobuild/configurable.rb

@@ -50,6 +50,17 @@ module Autobuild
             Autobuild.update_environment(prefix)
         end
 
+        def prepare_for_forced_build
+            FileUtils.rm_f buildstamp
+            FileUtils.rm_f configurestamp
+        end
+
+        def prepare_for_rebuild
+            if File.exists?(builddir) && builddir != srcdir
+                FileUtils.rm_rf builddir
+            end
+        end
+
         def depends_on(*packages)
             super
             stamps = packages.collect { |p| Package[p.to_s].installstamp }

data/lib/autobuild/package.rb

@@ -83,6 +83,7 @@ module Autobuild
 	def initialize(spec)
 	    @dependencies   = Array.new
 	    @provides       = Array.new
+            @parallel_build_level = nil
 
 	    if Hash === spec
 		name, depends = spec.to_a.first
@@ -127,6 +128,20 @@ module Autobuild
             @spec_dependencies = depends
 	end
 
+        # Called before a forced build. It should remove all the timestamp and
+        # target files so that all the build phases of this package gets
+        # retriggered. However, it should not clean the build products.
+        def prepare_for_forced_build
+            if File.exists?(installstamp)
+                FileUtils.rm_f installstamp
+            end
+        end
+
+        # Called when the user asked for a full rebuild. It should delete the
+        # build products so that a full build is retriggered.
+        def prepare_for_rebuild
+        end
+
         # Call the importer if there is one. Autodetection of "provides" should
         # be done there as well. See the documentation of Autobuild::Package for
         # more information.
@@ -279,6 +294,31 @@ module Autobuild
 	def self.[](name)
 	    @@packages[name.to_s] || @@provides[name.to_s]
 	end
+
+        # Sets the level of parallelism authorized while building this package
+        #
+        # See #parallel_build_level and Autobuild.parallel_build_level for more
+        # information.
+        #
+        # Note that not all package types use this value
+        def parallel_build_level=(value)
+            @parallel_build_level = Integer(value)
+        end
+
+        # Returns the level of parallelism authorized during the build for this
+        # particular package. If not set, defaults to the system-wide option 
+        # (Autobuild.parallel_build_level and Autobuild.parallel_build_level=).
+        #
+        # The default value is the number of CPUs on this system.
+        def parallel_build_level
+            if @parallel_build_level.nil?
+                Autobuild.parallel_build_level
+            elsif !@parallel_build_level || @parallel_build_level <= 0
+                1
+            else
+                @parallel_build_level
+            end
+        end
     end
 
     def self.package_set(spec)

data/lib/autobuild/packages/autotools.rb

@@ -62,7 +62,7 @@ module Autobuild
             doc_task do
                 Dir.chdir(builddir) do
                     Autobuild.progress "generating documentation for #{name}"
-                    Subprocess.run(name, 'doc', Autobuild.tool(:make), target)
+                    Subprocess.run(name, 'doc', Autobuild.tool(:make), "-j#{parallel_build_level}", target)
                     yield if block_given?
                 end
             end
@@ -233,7 +233,7 @@ module Autobuild
             Dir.chdir(builddir) {
                 Autobuild.progress "building #{name}"
                 Subprocess.run(name, 'build', './config.status')
-                Subprocess.run(name, 'build', Autobuild.tool(:make))
+                Subprocess.run(name, 'build', Autobuild.tool(:make), "-j#{parallel_build_level}")
             }
             Autobuild.touch_stamp(buildstamp)
         end
@@ -242,7 +242,7 @@ module Autobuild
         def install
             Dir.chdir(builddir) {
                 Autobuild.progress "installing #{name}"
-                Subprocess.run(name, 'install', Autobuild.tool(:make), 'install')
+                Subprocess.run(name, 'install', Autobuild.tool(:make), "-j#{parallel_build_level}", 'install')
             }
             Autobuild.touch_stamp(installstamp)
             Autobuild.update_environment(prefix)

data/lib/autobuild/packages/cmake.rb

@@ -40,7 +40,7 @@ module Autobuild
             doc_task do
                 Dir.chdir(builddir) do
                     Autobuild.progress "generating documentation for #{name}"
-                    Subprocess.run(name, 'doc', Autobuild.tool(:make), target)
+                    Subprocess.run(name, 'doc', Autobuild.tool(:make), "-j#{parallel_build_level}", target)
                     yield if block_given?
                 end
             end
@@ -134,11 +134,15 @@ module Autobuild
         # Do the build in builddir
         def build
             Dir.chdir(builddir) do
-                Autobuild.progress "building #{name}"
+                Autobuild.progress_with_value "building #{name}"
                 if always_reconfigure || !File.file?('Makefile')
                     Subprocess.run(name, 'build', Autobuild.tool(:cmake), '.')
                 end
-                Subprocess.run(name, 'build', Autobuild.tool(:make))
+                Subprocess.run(name, 'build', Autobuild.tool(:make), "-j#{parallel_build_level}") do |line|
+                    if line =~ /\[\s+(\d+)%\]/
+                        Autobuild.progress_value Integer($1)
+                    end
+                end
             end
             Autobuild.touch_stamp(buildstamp)
         end
@@ -147,7 +151,7 @@ module Autobuild
         def install
             Dir.chdir(builddir) do
                 Autobuild.progress "installing #{name}"
-                Subprocess.run(name, 'install', Autobuild.tool(:make), 'install')
+                Subprocess.run(name, 'build', Autobuild.tool(:make), "-j#{parallel_build_level}", 'install')
                 Autobuild.update_environment prefix
             end
             super

data/lib/autobuild/packages/dummy.rb

@@ -0,0 +1,31 @@
+module Autobuild
+    def self.dummy(spec)
+        ImporterPackage.new(spec)
+    end
+
+    class DummyPackage < Package
+        def installstamp
+            "#{srcdir}/#{STAMPFILE}"
+        end
+	
+        def initialize(*args)
+            super
+        end
+
+        def import
+        end
+
+        def prepare
+            %w{import prepare build doc}.each do |phase|
+                Rake.task("#{name}-#{phase}")
+                t = Rake::Task["#{name}-#{phase}"]
+                def t.needed?; false end
+            end
+            Rake.task(installstamp)
+            t = Rake::Task[installstamp]
+            def t.needed?; false end
+        end
+    end
+end
+    
+

data/lib/autobuild/reporting.rb

@@ -16,7 +16,23 @@ require 'autobuild/exceptions'
 
 module Autobuild
     def self.progress(msg)
-	puts "  #{msg}"
+        if @last_msg
+            progress_value(100)
+            puts
+        end
+        @last_msg = nil
+        puts "  #{msg}"
+    end
+    def self.progress_with_value(msg)
+        if @last_msg
+            progress_value(100)
+            puts
+        end
+        @last_msg = msg
+        print "  #{msg}"
+    end
+    def self.progress_value(value)
+        print "\r  #{@last_msg} (#{value}%)"
     end
 
     ## The reporting module provides the framework

data/lib/autobuild/subcommand.rb

@@ -1,5 +1,59 @@
 require 'autobuild/exceptions'
 require 'autobuild/reporting'
+require 'fcntl'
+
+module Autobuild
+    @parallel_build_level = nil
+    class << self
+        # Sets the level of parallelism during the build
+        #
+        # See #parallel_build_level for detailed information
+        attr_writer :parallel_build_level
+
+        # Returns the number of processes that can run in parallel during the
+        # build. This is a system-wide value that can be overriden in a
+        # per-package fashion by using Package#parallel_build_level.
+        #
+        # If not set, defaults to the number of CPUs on the system
+        #
+        # See also #parallel_build_level=
+        def parallel_build_level
+            if @parallel_build_level.nil?
+                # No user-set value, return the count of processors on this
+                # machine
+                autodetect_processor_count
+            elsif !@parallel_build_level || @parallel_build_level <= 0
+                1
+            else
+                @parallel_build_level
+            end
+        end
+    end
+
+    # Returns the number of CPUs present on this system
+    def self.autodetect_processor_count
+        if @processor_count
+            return @processor_count
+        end
+
+        if File.file?('/sys/devices/system/cpu/present')
+            range = File.read('/sys/devices/system/cpu/present').
+                chomp
+                
+            @processor_count = Integer(range.split('-').last) + 1
+        elsif File.file?('/proc/cpuinfo')
+            # Just count the numer of processor: \d lines
+            @processor_count = File.readlines('/proc/cpuinfo').
+                find_all { |l| l =~ /^processor\s+:\s+\d+$/ }.
+                count
+        else
+            # Hug... What kind of system is it ?
+            STDERR.puts "INFO: cannot autodetect the number of CPUs on this sytem"
+            @processor_count = 1
+        end
+    end
+end
+
 
 module Autobuild::Subprocess
     class Failed < Exception
@@ -12,6 +66,8 @@ module Autobuild::Subprocess
     CONTROL_COMMAND_NOT_FOUND = 1
     CONTROL_UNEXPECTED = 2
     def self.run(target, phase, *command)
+        STDOUT.sync = true
+
         # Filter nil and empty? in command
         command.reject!  { |o| o.nil? || (o.respond_to?(:empty?) && o.empty?) }
         command.collect! { |o| o.to_s }
@@ -41,17 +97,29 @@ module Autobuild::Subprocess
             logfile.puts "    #{command.join(" ")}"
 	    logfile.puts
 	    logfile.flush
+            logfile.sync = true
 
-            pread, pwrite = IO.pipe # to feed subprocess stdin 
+            if !input_streams.empty?
+                pread, pwrite = IO.pipe # to feed subprocess stdin 
+            end
+            if block_given? # the caller wants the stdout/stderr stream of the process, git it to him
+                outread, outwrite = IO.pipe
+                outread.sync = true
+                outwrite.sync = true
+            end
             cread, cwrite = IO.pipe # to control that exec goes well
 
+            cwrite.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+
             pid = fork do
                 cwrite.sync = true
                 begin
                     Process.setpriority(Process::PRIO_PROCESS, 0, Autobuild.nice)
-                    if Autobuild.verbose
-                        $stderr.dup.reopen(logfile.dup)
-                        $stdout.dup.reopen(logfile.dup)
+
+                    if outwrite
+                        outread.close
+                        $stderr.reopen(outwrite.dup)
+                        $stdout.reopen(outwrite.dup)
                     else
                         $stderr.reopen(logfile.dup)
                         $stdout.reopen(logfile.dup)
@@ -73,17 +141,19 @@ module Autobuild::Subprocess
             end
 
             # Feed the input
-            pread.close
-            begin
-                input_streams.each do |infile|
-                    File.open(infile) do |instream|
-                        instream.each_line { |line| pwrite.write(line) }
+            if !input_streams.empty?
+                pread.close
+                begin
+                    input_streams.each do |infile|
+                        File.open(infile) do |instream|
+                            instream.each_line { |line| pwrite.write(line) }
+                        end
                     end
+                rescue Errno::ENOENT => e
+                    raise Failed.new, "cannot open input files: #{e.message}"
                 end
-            rescue Errno::ENOENT => e
-                raise Failed.new, "cannot open input files: #{e.message}"
+                pwrite.close
             end
-            pwrite.close
 
             # Get control status
             cwrite.close
@@ -98,6 +168,22 @@ module Autobuild::Subprocess
                 end
             end
 
+            # If the caller asked for process output, provide it to him
+            # line-by-line.
+            if outread
+                outwrite.close
+                outread.each_line do |line|
+                    if Autobuild.verbose
+                        STDOUT.print line
+                    end
+                    logfile.puts line
+                    if block_given?
+                        yield(line)
+                    end
+                end
+                outread.close
+            end
+
             childpid, childstatus = Process.wait2(pid)
             childstatus
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: autobuild
 version: !ruby/object:Gem::Version 
-  version: 1.3.3
+  version: 1.4.0
 platform: ruby
 authors: 
 - Sylvain Joyeux
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-20 00:00:00 +02:00
+date: 2009-10-25 01:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -111,6 +111,7 @@ files:
 - lib/autobuild/package.rb
 - lib/autobuild/packages/autotools.rb
 - lib/autobuild/packages/cmake.rb
+- lib/autobuild/packages/dummy.rb
 - lib/autobuild/packages/genom.rb
 - lib/autobuild/packages/import.rb
 - lib/autobuild/packages/orogen.rb