autobuild 1.8.3 → 1.9.0.b1

data/lib/autobuild/package.rb

@@ -127,6 +127,9 @@ module Autobuild
         def updated?; !!@updated end
 
 	def initialize(spec = Hash.new)
+            @srcdir = @importdir = @logdir = nil
+            @update = nil
+            @failed = nil
 	    @dependencies   = Array.new
 	    @provides       = Array.new
             @parallel_build_level = nil
@@ -187,7 +190,7 @@ module Autobuild
         # target files so that all the build phases of this package gets
         # retriggered. However, it should not clean the build products.
         def prepare_for_forced_build
-            if File.exists?(installstamp)
+            if File.exist?(installstamp)
                 FileUtils.rm_f installstamp
             end
         end
@@ -197,7 +200,7 @@ module Autobuild
         def prepare_for_rebuild
             prepare_for_forced_build
 
-            if File.exists?(installstamp)
+            if File.exist?(installstamp)
                 FileUtils.rm_f installstamp
             end
         end
@@ -258,17 +261,22 @@ module Autobuild
         end
 
         # Call the importer if there is one. Autodetection of "provides" should
-        # be done there as well. See the documentation of Autobuild::Package for
-        # more information.
-	def import(only_local=false)
+        # be done there as well.
+        #
+        # (see Importer#import)
+	def import(options = Hash.new)
+            if !options.respond_to?(:to_hash)
+                options = Hash[only_local: options]
+            end
+
             if @importer
-                @importer.import(self,only_local)
-            elsif Autobuild.do_update
+                @importer.import(self, options)
+            elsif update?
                 message "%s: no importer defined, doing nothing"
             end
 
             # Add the dependencies declared in spec
-            depends_on *@spec_dependencies if @spec_dependencies
+            depends_on(*@spec_dependencies) if @spec_dependencies
             update_environment
         end
 
@@ -404,33 +412,6 @@ module Autobuild
             task
         end
 
-        module TaskExtension
-            attr_accessor :package
-        end
-
-        def source_tree(*args, &block)
-            task = Autobuild.source_tree(*args, &block)
-            task.extend TaskExtension
-            task.package = self
-            task
-        end
-
-        # Calls Rake to define a file task and then extends it with TaskExtension
-        def file(*args, &block)
-            task = super
-            task.extend TaskExtension
-            task.package = self
-            task
-        end
-
-        # Calls Rake to define a plain task and then extends it with TaskExtension
-        def task(*args, &block)
-            task = super
-            task.extend TaskExtension
-            task.package = self
-            task
-        end
-
         def doc_dir=(value); doc_utility.source_dir = value end
         def doc_dir; doc_utility.source_dir end
         def doc_target_dir=(value); doc_utility.target_dir = value end

data/lib/autobuild/packages/autotools.rb

@@ -102,9 +102,9 @@ module Autobuild
                 end
 
             if !programs.kind_of?(Hash)
-                programs = Array[*programs].inject({}) do |programs, spec|
-                    programs[spec.first] = spec.last
-                    programs
+                programs = Array[*programs].inject({}) do |progs, spec|
+                    progs[spec.first] = spec.last
+                    progs
                 end
             end
             programs.each do |name, opt|
@@ -168,7 +168,7 @@ module Autobuild
             # If it is not the case, remove it to force reconfiguration
 	    configureflags.flatten!
 	    force_reconfigure = false
-	    if File.exists?(configurestamp)
+	    if File.exist?(configurestamp)
 		output = IO.popen("#{configurestamp} --version").readlines.grep(/with options/).first.chomp
 		raise "invalid output of config.status --version" unless output =~ /with options "(.*)"$/
 		options = Shellwords.shellwords($1)
@@ -224,11 +224,11 @@ module Autobuild
             end
             using[:aclocal] = using[:autoconf] if using[:aclocal].nil?
             if using[:automake].nil?
-                using[:automake] = File.exists?(File.join(srcdir, 'Makefile.am'))
+                using[:automake] = File.exist?(File.join(srcdir, 'Makefile.am'))
             end
 
             if using[:libtool].nil?
-                using[:libtool] = File.exists?(File.join(srcdir, 'ltmain.sh'))
+                using[:libtool] = File.exist?(File.join(srcdir, 'ltmain.sh'))
             end
         end
 
@@ -237,7 +237,7 @@ module Autobuild
 	    conffile = "#{srcdir}/configure"
 	    if confsource
 		file conffile => confsource
-	    elsif confext = %w{.ac .in}.find { |ext| File.exists?("#{conffile}#{ext}") }
+	    elsif confext = %w{.ac .in}.find { |ext| File.exist?("#{conffile}#{ext}") }
 		file conffile => "#{conffile}#{confext}"
 	    else
 		raise PackageException.new(self, 'prepare'), "neither configure.ac nor configure.in present in #{srcdir}"
@@ -247,7 +247,7 @@ module Autobuild
                 isolate_errors do
                 in_dir(srcdir) do
                     if using[:autogen].nil?
-                        using[:autogen] = %w{autogen autogen.sh}.find { |f| File.exists?(File.join(srcdir, f)) }
+                        using[:autogen] = %w{autogen autogen.sh}.find { |f| File.exist?(File.join(srcdir, f)) }
                     end
 
                     autodetect_needed_stages

data/lib/autobuild/packages/cmake.rb

@@ -76,13 +76,14 @@ module Autobuild
         end
 
         def define(name, value)
-            @defines[name] = value
-        end
-
-        def doc_dir
-            if @doc_dir
-                File.expand_path(@doc_dir, builddir)
-            end
+            @defines[name] =
+                if value.respond_to?(:to_str)
+                    value.to_str
+                elsif value
+                    'ON'
+                else
+                    'OFF'
+                end
         end
 
         DOXYGEN_ACCEPTED_VARIABLES = {
@@ -265,13 +266,13 @@ module Autobuild
             # but no Makefile.
             #
             # Delete the CMakeCache to force reconfiguration
-            if !File.exists?( File.join(builddir, 'Makefile') )
+            if !File.exist?( File.join(builddir, 'Makefile') )
                 FileUtils.rm_f cmake_cache
             end
 
             doc_utility.source_ref_dir = builddir
 
-            if File.exists?(cmake_cache)
+            if File.exist?(cmake_cache)
                 all_defines = defines.dup
                 all_defines['CMAKE_INSTALL_PREFIX'] = prefix
                 all_defines['CMAKE_MODULE_PATH'] = "#{CMake.module_path.join(";")}"
@@ -361,24 +362,24 @@ module Autobuild
 
         # Do the build in builddir
         def build
+            current_message = String.new
             in_dir(builddir) do
                 progress_start "building %s" do
                     if always_reconfigure || !File.file?('Makefile')
                         Subprocess.run(self, 'build', Autobuild.tool(:cmake), '.')
                     end
 
-                    current_message = String.new
                     warning_count = 0
                     Autobuild.make_subcommand(self, 'build') do |line|
                         needs_display = false
                         if line =~ /\[\s*(\d+)%\]/
                             progress "building %s (#{Integer($1)}%)"
-                        elsif line !~ /^(?:Linking|Scanning|Building|Built)/
+                        elsif line !~ /^(?:Generating|Linking|Scanning|Building|Built)/
                             if line =~ /warning/
                                 warning_count += 1
                             end
                             if show_make_messages?
-                                current_message += line
+                                current_message += line + "\n"
                                 needs_display = true
                             end
                         end
@@ -400,6 +401,11 @@ module Autobuild
                 end
             end
             Autobuild.touch_stamp(buildstamp)
+        rescue ::Exception
+            current_message.split("\n").each do |l|
+                message "%s: #{l}", :magenta
+            end
+            raise
         end
 
         # Install the result in prefix

data/lib/autobuild/packages/genom.rb

@@ -117,7 +117,7 @@ module Autobuild
 
 	    # Check that the module has been generated with the same flags
 	    genom_mk = "#{srcdir}/autoconf/genom.mk"
-	    if File.exists?(genom_mk)
+	    if File.exist?(genom_mk)
 		contents = File.open(genom_mk).readlines
 		old_file = contents.find { |l| l =~ /^GENFILE/ }.gsub('GENFILE=', '').strip
 		old_flags = Shellwords.shellwords(

data/lib/autobuild/packages/gnumake.rb

@@ -18,31 +18,30 @@ module Autobuild
         make_is_gnumake?(path)
     end
 
-    def self.make_subcommand(pkg, phase, *options, &block)
+    def self.invoke_make_parallel(pkg, cmd_path = Autobuild.tool(:make))
         reserved = nil
-        cmd_path = Autobuild.tool(:make)
-        cmd = [cmd_path]
         if make_has_j_option?(cmd_path) && pkg.parallel_build_level != 1
             if manager = Autobuild.parallel_task_manager
                 job_server = manager.job_server
                 if !make_has_gnumake_jobserver?(cmd_path) || (pkg.parallel_build_level != Autobuild.parallel_build_level)
                     reserved = pkg.parallel_build_level
                     job_server.get(reserved - 1) # We already have one token taken by autobuild itself
-                    cmd << "-j#{pkg.parallel_build_level}"
-                else
-                    cmd << "--jobserver-fds=#{job_server.rio.fileno},#{job_server.wio.fileno}" << "-j"
+                    yield("-j#{pkg.parallel_build_level}")
                 end
-            else
-                cmd << "-j#{pkg.parallel_build_level}"
+                yield("--jobserver-fds=#{job_server.rio.fileno},#{job_server.wio.fileno}", "-j")
             end
+            yield("-j#{pkg.parallel_build_level}")
+        else yield
         end
-
-        cmd.concat(options)
-        Subprocess.run(pkg, phase, *cmd, &block)
-
     ensure
         if reserved
             job_server.put(reserved)
         end
     end
+    
+    def self.make_subcommand(pkg, phase, *options, &block)
+        invoke_make_parallel(pkg, Autobuild.tool(:make)) do |*make_parallel_options|
+            Subprocess.run(pkg, phase, Autobuild.tool(:make), *make_parallel_options, *options, &block)
+        end
+    end
 end

data/lib/autobuild/packages/orogen.rb

@@ -318,7 +318,7 @@ module Autobuild
             #
             # First, check if the command line changed
             needs_regen ||=
-                if File.exists?(genstamp)
+                if File.exist?(genstamp)
                     last_cmdline = File.read(genstamp).split("\n")
                     last_cmdline != cmdline
                 else

data/lib/autobuild/packages/ruby.rb

@@ -47,6 +47,14 @@ module Autobuild
             end
         end
 
+        def invoke_rake(setup_task = rake_setup_task)
+            if setup_task && File.file?(File.join(srcdir, 'Rakefile'))
+                Autobuild::Subprocess.run self, 'post-install',
+                    Autobuild.tool_in_path('ruby'), '-S', Autobuild.tool('rake'), setup_task,
+                    :working_directory => srcdir
+            end
+        end
+
         def install
             progress_start "setting up Ruby package %s", :done_message => 'set up Ruby package %s' do
                 Autobuild.update_environment srcdir
@@ -57,11 +65,7 @@ module Autobuild
                     Autobuild.env_add_path 'RUBYLIB', libdir
                 end
 
-                if rake_setup_task && File.file?(File.join(srcdir, 'Rakefile'))
-                    Autobuild::Subprocess.run self, 'post-install',
-                        Autobuild.tool_in_path('ruby'), '-S', Autobuild.tool('rake'), rake_setup_task,
-                        :working_directory => srcdir
-                end
+                invoke_rake
             end
             super
         end

data/lib/autobuild/reporting.rb

@@ -34,13 +34,17 @@ module Autobuild
     end
 
     def self.display_message(*args)
+        io = STDOUT
+        if args.last.kind_of?(IO)
+            io = args.pop
+        end
         msg =
             if args.empty? then ""
             else "#{color(*args)}"
             end
 
         if !Autobuild.progress_display_enabled?
-            puts msg
+            io.puts msg
             return
         end
 
@@ -50,7 +54,7 @@ module Autobuild
             end
 
         if !silent?
-            puts "\r#{msg}#{" " * [size - msg.size, 0].max}"
+            io.puts "\r#{msg}#{" " * [size - msg.size, 0].max}"
             if @last_progress_msg
                 print "#{@last_progress_msg}"
             end
@@ -64,12 +68,12 @@ module Autobuild
 
     # Displays an error message
     def self.error(message = "")
-        message("  ERROR: #{message}", :red, :bold)
+        message("  ERROR: #{message}", :red, :bold, STDERR)
     end
 
     # Displays a warning message
     def self.warn(message = "", *style)
-        message("  WARN: #{message}", :magenta, *style)
+        message("  WARN: #{message}", :magenta, *style, STDERR)
     end
 
     # @return [Boolean] true if there is some progress messages for the given
@@ -102,7 +106,7 @@ module Autobuild
                     progress(key, *options[:done_message])
                 end
                 progress_done(key, true)
-            rescue Exception => e
+            rescue Exception
                 progress_done(key, false)
                 raise
             end
@@ -335,7 +339,7 @@ module Autobuild
     ## Display using stdout
     class StdoutReporter < Reporter
         def error(error)
-            puts "Build failed: #{error}"
+            STDERR.puts "Build failed: #{error}"
         end
         def success
             puts "Build finished successfully at #{Time.now}"

data/lib/autobuild/subcommand.rb

@@ -39,11 +39,22 @@ module Autobuild
     reset_statistics
 
     @parallel_build_level = nil
+    @displayed_error_line_count = 10
     class << self
         # Sets the level of parallelism during the build
         #
         # See #parallel_build_level for detailed information
         attr_writer :parallel_build_level
+        
+        # set/get a value how much log lines should be displayed on errors
+        # this may be an integer or 'ALL' (which will be translated to -1) 
+        # this is not using an attr_accessor to be able to validate the values
+        def displayed_error_line_count=(value)
+            @displayed_error_line_count = validate_displayed_error_line_count(value)
+        end
+        def displayed_error_line_count
+            @displayed_error_line_count
+        end
 
         # Returns the number of processes that can run in parallel during the
         # build. This is a system-wide value that can be overriden in a
@@ -121,31 +132,82 @@ module Autobuild
 
         @processor_count
     end
+    
+    def self.validate_displayed_error_line_count(lines) 
+        if lines == 'ALL'
+            return Float::INFINITY
+        elsif lines.to_i > 0
+            return lines.to_i
+        end
+        raise ConfigError, 'Autobuild.displayed_error_line_count can only be a positive integer or \'ALL\'' 
+    end
+
 end
 
 
 module Autobuild::Subprocess
     class Failed < Exception
+        def retry?; @retry end
         attr_reader :status
-        def initialize(status = nil)
+
+        def initialize(status, do_retry)
             @status = status
+            @retry = do_retry
         end
     end
 
     CONTROL_COMMAND_NOT_FOUND = 1
     CONTROL_UNEXPECTED = 2
     CONTROL_INTERRUPT = 3
+
+    # Run a subcommand and return its standard output
+    #
+    # The command's standard and error outputs, as well as the full command line
+    # and an environment dump are saved in a log file in either the valure
+    # returned by target#logdir, or Autobuild.logdir if the target does not
+    # respond to #logdir.
+    #
+    # The subprocess priority is controlled by Autobuild.nice
+    #
+    # @param [String,(#name,#logdir,#working_directory)] target the target we
+    #   run the subcommand for. In general, it will be a Package object (run from
+    #   Package#run)
+    # @param [String] phase in which build phase this subcommand is executed
+    # @param [Array<String>] the command itself
+    # @yieldparam [String] line if a block is given, each output line from the
+    #   command's standard output are yield to it. This is meant for progress
+    #   display, and is disabled if Autobuild.verbose is set.
+    # @param [Hash] options
+    # @option options [String] :working_directory the directory in which the
+    #   command should be started. If nil, runs in the current directory. The
+    #   default is to either use the value returned by #working_directory on
+    #   {target} if it responds to it, or nil.
+    # @option options [Boolean] :retry (false) controls whether a failure to
+    #   execute this command should be retried by autobuild retry mechanisms (i.e.
+    #   in the importers) or not. {run} will not retry the command by itself, it
+    #   is passed as a hint for error handling clauses about whether the error
+    #   should be retried or not
+    # @option options [Array<IO>] :input_streams list of input streams that
+    #   should be fed to the command standard input. If a file needs to be given,
+    #   the :input argument can be used as well as a shortcut
+    # @option options [String] :input the path to a file whose content should be
+    #   fed to the command standard input
+    # @return [String] the command standard output
     def self.run(target, phase, *command)
         STDOUT.sync = true
 
         input_streams = []
-        options = Hash.new
+        options = Hash[retry: false]
         if command.last.kind_of?(Hash)
             options = command.pop
             options = Kernel.validate_options options,
-                :input => nil, :working_directory => nil
+                input: nil, working_directory: nil, retry: false,
+                input_streams: []
             if options[:input]
-                input_streams = [options[:input]]
+                input_streams << File.open(options[:input])
+            end
+            if options[:input_streams]
+                input_streams += options[:input_streams]
             end
         end
 
@@ -186,6 +248,7 @@ module Autobuild::Subprocess
         end
 
         Autobuild.register_logfile(logname)
+        subcommand_output = Array.new
 
         status = File.open(logname, open_flag) do |logfile|
             if Autobuild.keep_oldlogs
@@ -207,26 +270,20 @@ module Autobuild::Subprocess
             if !input_streams.empty?
                 pread, pwrite = IO.pipe # to feed subprocess stdin 
             end
-            if Autobuild.verbose || block_given? # the caller wants the stdout/stderr stream of the process, git it to him
-                outread, outwrite = IO.pipe
-                outread.sync = true
-                outwrite.sync = true
-            end
+
+            outread, outwrite = IO.pipe
+            outread.sync = true
+            outwrite.sync = true
+
             cread, cwrite = IO.pipe # to control that exec goes well
 
             if Autobuild.windows?
-                olddir = Dir.pwd
-                if options[:working_directory] && (options[:working_directory] != Dir.pwd)
-                    Dir.chdir(options[:working_directory])
-                end
-                system(*command)
-                result=$?.success?
-                if(!result)
-                    error = Autobuild::SubcommandFailed.new(target, command.join(" "), logname, "Systemcall")
-                    error.phase = phase
-                    raise error
+                Dir.chdir(options[:working_directory]) do
+                    if !system(*command)
+                        raise Failed.new($?.exitstatus, nil),
+                            "'#{command.join(' ')}' returned status #{status.exitstatus}"
+                    end
                 end
-                Dir.chdir(olddir)
                 return
             end
 
@@ -278,13 +335,12 @@ module Autobuild::Subprocess
             if !input_streams.empty?
                 pread.close
                 begin
-                    input_streams.each do |infile|
-                        File.open(infile) do |instream|
-                            instream.each_line { |line| pwrite.write(line) }
-                        end
+                    input_streams.each do |instream|
+                        instream.each_line { |line| pwrite.write(line) }
                     end
                 rescue Errno::ENOENT => e
-                    raise Failed.new, "cannot open input files: #{e.message}"
+                    raise Failed.new(nil, false),
+                        "cannot open input files: #{e.message}", retry: false
                 end
                 pwrite.close
             end
@@ -296,43 +352,45 @@ module Autobuild::Subprocess
                 # An error occured
                 value = value.unpack('I').first
                 if value == CONTROL_COMMAND_NOT_FOUND
-                    raise Failed.new, "command '#{command.first}' not found"
+                    raise Failed.new(nil, false),
+                        "command '#{command.first}' not found"
                 elsif value == CONTROL_INTERRUPT
                     raise Interrupt, "command '#{command.first}': interrupted by user"
                 else
-                    raise Failed.new, "something unexpected happened"
+                    raise Failed.new(nil, false),
+                        "something unexpected happened"
                 end
             end
 
             # If the caller asked for process output, provide it to him
             # line-by-line.
-            if outread
-                outwrite.close
-                outread.each_line do |line|
-                    if line.respond_to?(:force_encoding)
-                        line.force_encoding('BINARY')
-                    end
-                    if Autobuild.verbose
-                        STDOUT.print line
-                    end
-                    logfile.puts line
-                    # Do not yield the line if Autobuild.verbose is true, as it
-                    # would mix the progress output with the actual command
-                    # output. Assume that if the user wants the command output,
-                    # the autobuild progress output is unnecessary
-                    if !Autobuild.verbose && block_given?
-                        yield(line)
-                    end
+            outwrite.close
+            outread.each_line do |line|
+                line.force_encoding('BINARY')
+                line = line.chomp
+                subcommand_output << line
+
+                if Autobuild.verbose
+                    STDOUT.puts line
+                end
+                logfile.puts line
+                # Do not yield the line if Autobuild.verbose is true, as it
+                # would mix the progress output with the actual command
+                # output. Assume that if the user wants the command output,
+                # the autobuild progress output is unnecessary
+                if !Autobuild.verbose && block_given?
+                    yield(line)
                 end
-                outread.close
             end
+            outread.close
 
-            childpid, childstatus = Process.wait2(pid)
+            _, childstatus = Process.wait2(pid)
             childstatus
         end
 
         if !status.exitstatus || status.exitstatus > 0
-            raise Failed.new(status.exitstatus), "'#{command.join(' ')}' returned status #{status.exitstatus}"
+            raise Failed.new(status.exitstatus, nil),
+                "'#{command.join(' ')}' returned status #{status.exitstatus}"
         end
 
         duration = Time.now - start_time
@@ -345,14 +403,16 @@ module Autobuild::Subprocess
         if target.respond_to?(:add_stat)
             target.add_stat(phase, duration)
         end
+        subcommand_output
 
     rescue Failed => e
-        error = Autobuild::SubcommandFailed.new(target, command.join(" "), logname, e.status)
+        error = Autobuild::SubcommandFailed.new(target, command.join(" "), logname, e.status, subcommand_output)
+        error.retry = if e.retry?.nil? then options[:retry]
+                      else e.retry?
+                      end
         error.phase = phase
         raise error, e.message
     end
 
 end
 
-
-