autobuild 1.18.1 → 1.22.0

data/lib/autobuild/packages/autotools.rb

@@ -25,13 +25,8 @@ module Autobuild
     # To override this default behaviour on a per-package basis, use Autotools#use
     #
     class Autotools < Configurable
-        attr_accessor   :using
-        attr_accessor   :configureflags
-        attr_accessor   :aclocal_flags
-        attr_accessor   :autoheader_flags
-        attr_accessor   :autoconf_flags
-        attr_accessor   :automake_flags
-        attr_accessor   :bear_flags
+        attr_accessor :using, :configureflags, :aclocal_flags, :autoheader_flags,
+                      :autoconf_flags, :automake_flags, :bear_flags
 
         @builddir = 'build'
         @@enable_bear_globally = false
@@ -278,7 +273,7 @@ module Autobuild
                 file conffile => "#{conffile}#{confext}"
             elsif using[:autoconf]
                 raise PackageException.new(self, 'prepare'),
-                    "neither configure.ac nor configure.in present in #{srcdir}"
+                      "neither configure.ac nor configure.in present in #{srcdir}"
             end
 
             file conffile do

data/lib/autobuild/packages/cmake.rb

@@ -25,6 +25,7 @@ module Autobuild
             end
 
             attr_writer :full_reconfigures
+
             def full_reconfigures?
                 @full_reconfigures
             end
@@ -36,8 +37,7 @@ module Autobuild
             # It can be overriden on a per-package basis with CMake.generator=
             attr_accessor :generator
 
-            attr_reader :prefix_path
-            attr_reader :module_path
+            attr_reader :prefix_path, :module_path
 
             # Whether files that are not within CMake's install manifest but are
             # present in the prefix should be deleted. Note that the contents of
@@ -63,6 +63,7 @@ module Autobuild
 
         # a key => value association of defines for CMake
         attr_reader :defines
+
         # The list of all -D options that should be passed on to CMake
         def all_defines
             additional_defines = Hash[
@@ -81,6 +82,7 @@ module Autobuild
         # Sets a generator explicitely for this component. See #generator and
         # CMake.generator
         attr_writer :generator
+
         # The CMake generator to use. You must choose one that generates
         # Makefiles. If not set for this package explicitely, it is using the
         # global value CMake.generator.
@@ -267,7 +269,7 @@ module Autobuild
             run('doc', Autobuild.tool(:doxygen), doxyfile)
         end
 
-        def common_utility_handling(utility, target, start_msg, done_msg)
+        def common_utility_handling(utility, target, *args, start_msg, done_msg)
             utility.source_ref_dir = builddir
             utility.task do
                 progress_start start_msg, :done_message => done_msg do
@@ -277,7 +279,7 @@ module Autobuild
                         run(utility.name,
                             Autobuild.tool(:make),
                             "-j#{parallel_build_level}",
-                            target,
+                            target, *args,
                             working_directory: builddir)
                     end
                     yield if block_given?
@@ -295,7 +297,7 @@ module Autobuild
 
         def with_tests(target = 'test', &block)
             common_utility_handling(
-                test_utility, target,
+                test_utility, target, "ARGS=-V",
                 "running tests for %s",
                 "successfully ran tests for %s", &block)
         end
@@ -320,7 +322,7 @@ module Autobuild
             end
         end
 
-        def import(options = Hash.new)
+        def import(**options)
             super
 
             Dir.glob(File.join(srcdir, "*.pc.in")) do |file|
@@ -410,7 +412,7 @@ module Autobuild
                 in_dir(builddir) do
                     unless File.file?(File.join(srcdir, 'CMakeLists.txt'))
                         raise ConfigException.new(self, 'configure'),
-                            "#{srcdir} contains no CMakeLists.txt file"
+                              "#{srcdir} contains no CMakeLists.txt file"
                     end
 
                     command = ["cmake"]
@@ -542,5 +544,12 @@ module Autobuild
                 message "%s: removed #{counter} obsolete files from prefix (cmake)"
             end
         end
+
+        def self_fingerprint
+            return unless (base = super)
+
+            all_defines = self.class.defines.merge(defines).sort_by(&:first)
+            Digest::SHA1.hexdigest(base + all_defines.join(""))
+        end
     end
 end

data/lib/autobuild/packages/dummy.rb

@@ -8,10 +8,6 @@ module Autobuild
             "#{srcdir}/#{STAMPFILE}"
         end
 
-        def initialize(*args)
-            super
-        end
-
         def import(options = Hash.new); end
 
         def prepare

data/lib/autobuild/packages/gnumake.rb

@@ -95,7 +95,7 @@ module Autobuild
     def self.make_subcommand(pkg, phase, *options, &block)
         invoke_make_parallel(pkg, Autobuild.tool(:make)) do |*make_parallel_options|
             pkg.run(phase, Autobuild.tool(:make),
-                *make_parallel_options, *options, &block)
+                    *make_parallel_options, *options, &block)
         end
     end
 end

data/lib/autobuild/packages/orogen.rb

@@ -97,19 +97,19 @@ module Autobuild
             end
         end
 
-        attr_writer :corba
+        attr_writer :corba, :orogen_file
+
         def corba
             @corba || (@corba.nil? && Orogen.corba)
         end
 
         # Overrides the global Orocos.extended_states for this particular package
         attr_writer :extended_states
+
         def extended_states
             @extended_states || (@extended_states.nil? && Orogen.extended_states)
         end
 
-        attr_writer :orogen_file
-
         # Path to the orogen file used for this package
         #
         # If not set, the class will look for a .orogen file in the package
@@ -128,7 +128,7 @@ module Autobuild
                     return File.basename(path)
                 end
                 raise ArgumentError,
-                    "cannot find an oroGen specification file in #{srcdir}"
+                      "cannot find an oroGen specification file in #{srcdir}"
             end
         end
 
@@ -235,6 +235,13 @@ module Autobuild
                 cmdline << "--type-export-policy=#{Orogen.default_type_export_policy}"
                 cmdline << "--transports=#{Orogen.transports.sort.uniq.join(',')}"
             end
+            if version >= "1.2"
+                cmdline << "--parallel-codegen=#{parallel_build_level}"
+                if (job_server = Autobuild.parallel_task_manager&.job_server)
+                    fds = "#{job_server.rio.fileno},#{job_server.wio.fileno}"
+                    cmdline << "--jobserver-auth=#{fds}"
+                end
+            end
 
             # Now, add raw options
             #

data/lib/autobuild/packages/pkgconfig.rb

@@ -2,8 +2,7 @@ require 'autobuild/pkgconfig'
 
 module Autobuild
     class InstalledPkgConfig < Package
-        attr_reader :pkgconfig
-        attr_reader :prefix
+        attr_reader :pkgconfig, :prefix
 
         def initialize(name)
             @pkgconfig = PkgConfig.new(name)
@@ -23,6 +22,7 @@ module Autobuild
             pcfile
         end
     end
+
     def self.installed_pkgconfig(name, &block)
         InstalledPkgConfig.new(name, &block)
     end

data/lib/autobuild/packages/python.rb

@@ -9,8 +9,7 @@ module Autobuild
 
     # Handler class to build python-based packages
     class Python < Configurable
-        attr_accessor   :buildflags
-        attr_accessor   :installflags
+        attr_accessor :buildflags, :installflags
 
         def configurestamp
             "#{builddir}/configure-autobuild-stamp"
@@ -22,9 +21,8 @@ module Autobuild
             super
         end
 
-        def prepare
-            super
-            @install_mode = File.file?(File.join(srcdir, 'setup.py'))
+        def install_mode?
+            File.file?(File.join(srcdir, 'setup.py'))
         end
 
         def prepare_for_forced_build
@@ -65,7 +63,7 @@ module Autobuild
 
         # Do the build in builddir
         def build
-            return unless @install_mode
+            return unless install_mode?
 
             command = generate_build_command
             command << '--force' if @forced
@@ -78,7 +76,7 @@ module Autobuild
 
         # Install the result in prefix
         def install
-            return unless @install_mode
+            return unless install_mode?
 
             command = generate_install_command
             command << '--force' if @forced
@@ -91,7 +89,7 @@ module Autobuild
 
         def update_environment
             super
-            path = @install_mode ? python_path : srcdir
+            path = install_mode? ? python_path : srcdir
             env_add_path 'PYTHONPATH', path
         end
     end

data/lib/autobuild/packages/ruby.rb

@@ -40,7 +40,7 @@ module Autobuild
                                done_message: 'generated documentation for %s' do
                     run 'doc',
                         Autobuild.tool_in_path('ruby'), '-S',
-                            Autobuild.tool('rake'), rake_doc_task,
+                        Autobuild.tool('rake'), rake_doc_task,
                         working_directory: srcdir
                 end
             end
@@ -52,7 +52,7 @@ module Autobuild
                                done_message: 'tests passed for %s' do
                     run 'test',
                         Autobuild.tool_in_path('ruby'), '-S',
-                            Autobuild.tool('rake'), rake_test_task, *rake_test_options,
+                        Autobuild.tool('rake'), rake_test_task, *rake_test_options,
                         working_directory: srcdir
                 end
             end
@@ -62,7 +62,7 @@ module Autobuild
             if setup_task && File.file?(File.join(srcdir, 'Rakefile'))
                 run 'post-install',
                     Autobuild.tool_in_path('ruby'), '-S',
-                        Autobuild.tool('rake'), setup_task,
+                    Autobuild.tool('rake'), setup_task,
                     working_directory: srcdir
             end
         end
@@ -80,7 +80,7 @@ module Autobuild
             %w[ext tmp].each do |extdir|
                 if File.directory?(extdir)
                     Find.find(extdir) do |file|
-                        next if file !~ /\<Makefile\>|\<CMakeCache.txt\>$/
+                        next if file !~ /<Makefile>|<CMakeCache.txt>$/
 
                         FileUtils.rm_rf file
                     end
@@ -94,7 +94,7 @@ module Autobuild
                 begin
                     run 'clean',
                         Autobuild.tool_in_path('ruby'), '-S',
-                            Autobuild.tool('rake'), rake_clean_task,
+                        Autobuild.tool('rake'), rake_clean_task,
                         working_directory: srcdir
                 rescue Autobuild::SubcommandFailed => e
                     warn "%s: clean failed. If this package does not need a clean target,"

data/lib/autobuild/parallel.rb

@@ -4,15 +4,10 @@ module Autobuild
     # Since autobuild does not use task arguments, we don't support them for
     # simplicity
     class RakeTaskParallelism
-        attr_reader :available_workers
-        attr_reader :finished_workers
-        attr_reader :workers
-
-        attr_reader :job_server
+        attr_reader :available_workers, :finished_workers, :workers, :job_server
 
         class JobServer
-            attr_reader :rio
-            attr_reader :wio
+            attr_reader :rio, :wio
 
             def initialize(level)
                 @rio, @wio = IO.pipe
@@ -76,15 +71,12 @@ module Autobuild
         end
 
         class ProcessingState
-            attr_reader :reverse_dependencies
-            attr_reader :processed
-            attr_reader :started_packages
-            attr_reader :active_tasks
-            attr_reader :queue
-            attr_reader :priorities
-
-            def initialize(reverse_dependencies)
+            attr_reader :reverse_dependencies, :processed, :started_packages,
+                        :active_tasks, :queue, :priorities
+
+            def initialize(reverse_dependencies, completion_callback: proc {})
                 @reverse_dependencies = reverse_dependencies
+                @completion_callback = completion_callback
                 @processed = Set.new
                 @active_tasks = Set.new
                 @priorities = Hash.new
@@ -143,6 +135,8 @@ module Autobuild
                         push(candidate, priorities[task])
                     end
                 end
+
+                @completion_callback.call(task)
             end
 
             def trivial_task?(task)
@@ -153,7 +147,7 @@ module Autobuild
 
         # Invokes the provided tasks. Unlike the rake code, this is a toplevel
         # algorithm that does not use recursion
-        def invoke_parallel(required_tasks)
+        def invoke_parallel(required_tasks, completion_callback: proc {})
             tasks = Set.new
             reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }
             required_tasks.each do |t|
@@ -162,7 +156,8 @@ module Autobuild
             # The queue is the set of tasks for which all prerequisites have
             # been successfully executed (or where not needed). I.e. it is the
             # set of tasks that can be queued for execution.
-            state = ProcessingState.new(reverse_dependencies)
+            state = ProcessingState.new(reverse_dependencies,
+                                        completion_callback: completion_callback)
             tasks.each do |t|
                 state.push(t) if state.ready?(t)
             end
@@ -188,12 +183,16 @@ module Autobuild
                     break if !pending_task && available_workers.size == workers.size
                 end
 
-                if state.trivial_task?(pending_task)
-                    Worker.execute_task(pending_task)
+                bypass_task = pending_task.disabled? ||
+                              pending_task.already_invoked? ||
+                              !pending_task.needed?
+
+                if bypass_task
+                    pending_task.already_invoked = true
                     state.process_finished_task(pending_task)
                     next
-                elsif pending_task.already_invoked? || !pending_task.needed?
-                    pending_task.already_invoked = true
+                elsif state.trivial_task?(pending_task)
+                    Worker.execute_task(pending_task)
                     state.process_finished_task(pending_task)
                     next
                 end

data/lib/autobuild/pkgconfig.rb

@@ -5,6 +5,7 @@ class PkgConfig
 
         def initialize(name)
             @name = name
+            super()
         end
 
         def to_s

data/lib/autobuild/progress_display.rb

@@ -1,3 +1,6 @@
+require "concurrent/atomic/atomic_boolean"
+require "concurrent/array"
+
 module Autobuild
     # Management of the progress display
     class ProgressDisplay
@@ -5,20 +8,72 @@ module Autobuild
             @io = io
             @cursor = TTY::Cursor
             @last_formatted_progress = []
-            @progress_messages = []
+            @progress_messages = Concurrent::Array.new
 
             @silent = false
             @color = color
-            @progress_enabled = true
             @display_lock = Mutex.new
+
+            @next_progress_display = Time.at(0)
+            @progress_mode = :single_line
+            @progress_period = 0.1
+
+            @message_queue = Queue.new
+            @forced_progress_display = Concurrent::AtomicBoolean.new(false)
         end
 
-        attr_writer :silent
+        def synchronize(&block)
+            result = @display_lock.synchronize(&block)
+            refresh_display
+            result
+        end
+
+        # Set the minimum time between two progress messages
+        #
+        # @see period
+        def progress_period=(period)
+            @progress_period = Float(period)
+        end
+
+        # Minimum time between two progress displays
+        #
+        # This does not affect normal messages
+        #
+        # @return [Float]
+        attr_reader :progress_period
+
+        # Valid progress modes
+        #
+        # @see progress_mode=
+        PROGRESS_MODES = %I[single_line newline off].freeze
+
+        # Sets how progress messages will be displayed
+        #
+        # @param [String] the new mode. Can be either 'single_line', where a
+        #   progress message replaces the last one, 'newline' which displays
+        #   each on a new line or 'off' to disable progress messages altogether
+        def progress_mode=(mode)
+            mode = mode.to_sym
+            unless PROGRESS_MODES.include?(mode)
+                raise ArgumentError,
+                      "#{mode} is not a valid mode, expected one of "\
+                      "#{PROGRESS_MODES.join(', ')}"
+            end
+            @progress_mode = mode
+        end
+
+        # Return the current display mode
+        #
+        # @return [Symbol]
+        # @see mode=
+        attr_reader :progress_mode
 
         def silent?
             @silent
         end
 
+        attr_writer :silent
+
         def silent
             silent = @silent
             @silent = true
@@ -27,24 +82,23 @@ module Autobuild
             @silent = silent
         end
 
-        attr_writer :progress_enabled
+        # @deprecated use progress_mode= instead
+        def progress_enabled=(flag)
+            self.progress_mode = flag ? :single_line : :off
+        end
 
+        # Whether progress messages will be displayed at all
         def progress_enabled?
-            !@silent && @progress_enabled
+            !@silent && (@progress_mode != :off)
         end
 
         def message(message, *args, io: @io, force: false)
             return if silent? && !force
 
             io = args.pop if args.last.respond_to?(:to_io)
+            @message_queue << [message, args, io]
 
-            @display_lock.synchronize do
-                io.print "#{@cursor.column(1)}#{@cursor.clear_screen_down}"\
-                    "#{@color.call(message, *args)}\n"
-                io.flush if @io != io
-                display_progress
-                @io.flush
-            end
+            refresh_display
         end
 
         def progress_start(key, *args, done_message: nil)
@@ -53,13 +107,13 @@ module Autobuild
             formatted_message = @color.call(*args)
             @progress_messages << [key, formatted_message]
             if progress_enabled?
-                @display_lock.synchronize do
-                    display_progress
-                end
+                @forced_progress_display.make_true
             else
                 message "  #{formatted_message}"
             end
 
+            refresh_display
+
             if block_given?
                 begin
                     result = yield
@@ -73,72 +127,99 @@ module Autobuild
         end
 
         def progress(key, *args)
-            @display_lock.synchronize do
-                found = false
-                @progress_messages.map! do |msg_key, msg|
-                    if msg_key == key
-                        found = true
-                        [msg_key, @color.call(*args)]
-                    else
-                        [msg_key, msg]
-                    end
+            found = false
+            @progress_messages.map! do |msg_key, msg|
+                if msg_key == key
+                    found = true
+                    [msg_key, @color.call(*args)]
+                else
+                    [msg_key, msg]
                 end
-                @progress_messages << [key, @color.call(*args)] unless found
-                display_progress
             end
+            @progress_messages << [key, @color.call(*args)] unless found
+
+            refresh_display
         end
 
         def progress_done(key, display_last = true, message: nil)
-            changed = @display_lock.synchronize do
-                current_size = @progress_messages.size
-                @progress_messages.delete_if do |msg_key, msg|
-                    if msg_key == key
-                        message = msg if display_last && !message
-                        true
-                    end
+            current_size = @progress_messages.size
+            @progress_messages.delete_if do |msg_key, msg|
+                if msg_key == key
+                    message = msg if display_last && !message
+                    true
                 end
-                current_size != @progress_messages.size
             end
+            changed = current_size != @progress_messages.size
 
             if changed
                 if message
                     message("  #{message}")
-                    # Note: message calls display_progress already
+                    # NOTE: message updates the display already
                 else
-                    @display_lock.synchronize do
-                        display_progress
-                    end
+                    refresh_display
                 end
                 true
             end
         end
 
-        def display_progress
+        def refresh_display
+            return unless @display_lock.try_lock
+
+            begin
+                refresh_display_under_lock
+            ensure
+                @display_lock.unlock
+            end
+        end
+
+        def refresh_display_under_lock
+            # Display queued messages
+            until @message_queue.empty?
+                message, args, io = @message_queue.pop
+                io.print @cursor.clear_screen_down if @progress_mode == :single_line
+                io.puts @color.call(message, *args)
+
+                io.flush if @io != io
+            end
+
+            # And re-display the progress
+            display_progress(consider_period: @forced_progress_display.false?)
+            @forced_progress_display.make_false
+            @io.flush
+        end
+
+        def display_progress(consider_period: true)
             return unless progress_enabled?
+            return if consider_period && (@next_progress_display > Time.now)
 
-            formatted = format_grouped_messages(@progress_messages.map(&:last),
-                indent: "  ")
-            @io.print @cursor.clear_screen_down
-            @io.print formatted.join("\n")
-            if formatted.size > 1
-                @io.print "#{@cursor.up(formatted.size - 1)}#{@cursor.column(0)}"
+            formatted = format_grouped_messages(
+                @progress_messages.map(&:last),
+                indent: "  "
+            )
+            if @progress_mode == :newline
+                @io.print formatted.join("\n")
+                @io.print "\n"
             else
+                @io.print @cursor.clear_screen_down
+                @io.print formatted.join("\n")
+                @io.print @cursor.up(formatted.size - 1) if formatted.size > 1
                 @io.print @cursor.column(0)
             end
             @io.flush
+            @next_progress_display = Time.now + @progress_period
         end
 
         def find_common_prefix(msg, other_msg)
-            msg = msg.split(" ")
-            other_msg = other_msg.split(" ")
+            msg = msg.split(' ')
+            other_msg = other_msg.split(' ')
             msg.each_with_index do |token, idx|
                 if other_msg[idx] != token
                     prefix = msg[0..(idx - 1)].join(" ")
-                    prefix << " " unless prefix.empty?
+                    prefix << ' ' unless prefix.empty?
                     return prefix
                 end
             end
-            msg.join(" ")
+            msg.join(' ')
         end
 
         def group_messages(messages)