autobuild 1.17.0 → 1.21.0

data/lib/autobuild/parallel.rb

@@ -1,5 +1,3 @@
-require 'thread'
-
 module Autobuild
     # This is a rewrite of the Rake task invocation code to use parallelism
     #
@@ -23,9 +21,11 @@ def initialize(level)
                 wio.fcntl(Fcntl::F_SETFD, 0)
                 put(level)
             end
+
             def get(token_count = 1)
                 @rio.read(token_count)
             end
+
             def put(token_count = 1)
                 @wio.write(" " * token_count)
             end
@@ -36,7 +36,6 @@ def initialize(level = Autobuild.parallel_build_level)
             @available_workers = Array.new
             @finished_workers = Queue.new
             @workers = Array.new
-
         end
 
         def wait_for_worker_to_end(state)
@@ -46,9 +45,12 @@ def wait_for_worker_to_end(state)
             if error
                 if available_workers.size != workers.size
                     if finished_task.respond_to?(:package) && finished_task.package
-                        Autobuild.error "got an error processing #{finished_task.package.name}, waiting for pending jobs to end"
+                        Autobuild.error "got an error processing "\
+                            "#{finished_task.package.name}, "\
+                            "waiting for pending jobs to end"
                     else
-                        Autobuild.error "got an error doing parallel processing, waiting for pending jobs to end"
+                        Autobuild.error "got an error doing parallel processing, "\
+                            "waiting for pending jobs to end"
                     end
                 end
                 begin
@@ -61,16 +63,14 @@ def wait_for_worker_to_end(state)
             state.process_finished_task(finished_task)
         end
 
-        def discover_dependencies(all_tasks, reverse_dependencies, t)
-            if t.already_invoked?
-                return 
-            end
+        def discover_dependencies(all_tasks, reverse_dependencies, task)
+            return if task.already_invoked?
+            return if all_tasks.include?(task) # already discovered or being discovered
 
-            return if all_tasks.include?(t) # already discovered or being discovered
-            all_tasks << t
+            all_tasks << task
 
-            t.prerequisite_tasks.each do |dep_t|
-                reverse_dependencies[dep_t] << t
+            task.prerequisite_tasks.each do |dep_t|
+                reverse_dependencies[dep_t] << task
                 discover_dependencies(all_tasks, reverse_dependencies, dep_t)
             end
         end
@@ -83,8 +83,9 @@ class ProcessingState
             attr_reader :queue
             attr_reader :priorities
 
-            def initialize(reverse_dependencies)
+            def initialize(reverse_dependencies, completion_callback: proc { })
                 @reverse_dependencies = reverse_dependencies
+                @completion_callback = completion_callback
                 @processed = Set.new
                 @active_tasks = Set.new
                 @priorities = Hash.new
@@ -101,7 +102,7 @@ def push(task, base_priority = 1)
             end
 
             def find_task
-                if task = queue.sort_by { |t, p| p }.first
+                if (task = queue.min_by { |_t, p| p })
                     priorities[task.first] = task.last
                     task.first
                 end
@@ -143,16 +144,19 @@ def process_finished_task(task)
                         push(candidate, priorities[task])
                     end
                 end
+
+                @completion_callback.call(task)
             end
 
             def trivial_task?(task)
-                (task.kind_of?(Autobuild::SourceTreeTask) || task.kind_of?(Rake::FileTask)) && task.actions.empty?
+                (task.kind_of?(Autobuild::SourceTreeTask) ||
+                    task.kind_of?(Rake::FileTask)) && task.actions.empty?
             end
         end
 
         # Invokes the provided tasks. Unlike the rake code, this is a toplevel
         # algorithm that does not use recursion
-        def invoke_parallel(required_tasks)
+        def invoke_parallel(required_tasks, completion_callback: proc { })
             tasks = Set.new
             reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }
             required_tasks.each do |t|
@@ -161,13 +165,12 @@ def invoke_parallel(required_tasks)
             # The queue is the set of tasks for which all prerequisites have
             # been successfully executed (or where not needed). I.e. it is the
             # set of tasks that can be queued for execution.
-            state = ProcessingState.new(reverse_dependencies)
+            state = ProcessingState.new(reverse_dependencies,
+                                        completion_callback: completion_callback)
             tasks.each do |t|
-                if state.ready?(t)
-                    state.push(t)
-                end
+                state.push(t) if state.ready?(t)
             end
-            
+
             # Build a reverse dependency graph (i.e. a mapping from a task to
             # the tasks that depend on it)
 
@@ -175,9 +178,9 @@ def invoke_parallel(required_tasks)
             # topological sort since we would then have to scan all tasks each
             # time for tasks that have no currently running prerequisites
 
-            while true
+            loop do
                 pending_task = state.pop
-                if !pending_task
+                unless pending_task
                     # If we have pending workers, wait for one to be finished
                     # until either they are all finished or the queue is not
                     # empty anymore
@@ -186,17 +189,19 @@ def invoke_parallel(required_tasks)
                         pending_task = state.pop
                     end
 
-                    if !pending_task && available_workers.size == workers.size
-                        break
-                    end
+                    break if !pending_task && available_workers.size == workers.size
                 end
 
-                if state.trivial_task?(pending_task)
-                    Worker.execute_task(pending_task)
+                bypass_task = pending_task.disabled? ||
+                              pending_task.already_invoked? ||
+                              !pending_task.needed?
+
+                if bypass_task
+                    pending_task.already_invoked = true
                     state.process_finished_task(pending_task)
                     next
-                elsif pending_task.already_invoked? || !pending_task.needed?
-                    pending_task.already_invoked = true
+                elsif state.trivial_task?(pending_task)
+                    Worker.execute_task(pending_task)
                     state.process_finished_task(pending_task)
                     next
                 end
@@ -204,9 +209,7 @@ def invoke_parallel(required_tasks)
                 # Get a job server token
                 job_server.get
 
-                while !finished_workers.empty?
-                    wait_for_worker_to_end(state)
-                end
+                wait_for_worker_to_end(state) until finished_workers.empty?
 
                 # We do have a job server token, so we are allowed to allocate a
                 # new worker if none are available
@@ -222,9 +225,9 @@ def invoke_parallel(required_tasks)
             end
 
             not_processed = tasks.find_all { |t| !t.already_invoked? }
-            if !not_processed.empty?
+            unless not_processed.empty?
                 cycle = resolve_cycle(tasks, not_processed, reverse_dependencies)
-                raise "cycle in task graph: #{cycle.map(&:name).sort.join(", ")}"
+                raise "cycle in task graph: #{cycle.map(&:name).sort.join(', ')}"
             end
         end
 
@@ -232,7 +235,7 @@ def resolve_cycle(all_tasks, tasks, reverse_dependencies)
             cycle = tasks.dup
             chain = []
             next_task = tasks.first
-            while true
+            loop do
                 task = next_task
                 chain << task
                 tasks.delete(next_task)
@@ -244,10 +247,12 @@ def resolve_cycle(all_tasks, tasks, reverse_dependencies)
                         true
                     end
                 end
-                if !next_task
-                    Autobuild.fatal "parallel processing stopped prematurely, but no cycle is present in the remaining tasks"
-                    Autobuild.fatal "remaining tasks: #{cycle.map(&:name).join(", ")}"
-                    Autobuild.fatal "known dependencies at initialization time that could block the processing of the remaining tasks"
+                unless next_task
+                    Autobuild.fatal "parallel processing stopped prematurely, "\
+                        "but no cycle is present in the remaining tasks"
+                    Autobuild.fatal "remaining tasks: #{cycle.map(&:name).join(', ')}"
+                    Autobuild.fatal "known dependencies at initialization time that "\
+                        "could block the processing of the remaining tasks"
                     reverse_dependencies.each do |parent_task, parents|
                         if cycle.include?(parent_task)
                             parents.each do |p|
@@ -255,13 +260,14 @@ def resolve_cycle(all_tasks, tasks, reverse_dependencies)
                             end
                         end
                     end
-                    Autobuild.fatal "known dependencies right now that could block the processing of the remaining tasks"
+                    Autobuild.fatal "known dependencies right now that could block "\
+                        "the processing of the remaining tasks"
                     all_tasks.each do |p|
                         (cycle & p.prerequisite_tasks).each do |t|
                             Autobuild.fatal "  #{p}: #{t}"
                         end
                     end
-                    raise "failed to resolve cycle in #{cycle.map(&:name).join(", ")}"
+                    raise "failed to resolve cycle in #{cycle.map(&:name).join(', ')}"
                 end
             end
             chain
@@ -301,7 +307,7 @@ def do_task(task)
             end
 
             def last_result
-                return @last_finished_task, @last_error
+                [@last_finished_task, @last_error]
             end
 
             def queue(task)
@@ -321,5 +327,3 @@ class << self
         attr_accessor :parallel_task_manager
     end
 end
-
-

data/lib/autobuild/pkgconfig.rb

@@ -2,10 +2,16 @@
 class PkgConfig
     class NotFound < RuntimeError
         attr_reader :name
-        def initialize(name); @name = name end
-        def to_s; "#{name} is not available to pkg-config" end
+
+        def initialize(name)
+            @name = name
+        end
+
+        def to_s
+            "#{name} is not available to pkg-config"
+        end
     end
-    
+
     # The module name
     attr_reader :name
     # The module version
@@ -14,30 +20,36 @@ def to_s; "#{name} is not available to pkg-config" end
     # Create a PkgConfig object for the package +name+
     # Raises PkgConfig::NotFound if the module does not exist
     def initialize(name)
-        if !system("pkg-config --exists #{name}")
-            raise NotFound.new(name)
+        unless system("pkg-config --exists #{name}")
+            raise NotFound.new(name), "pkg-config package '#{name}' not found"
         end
-        
+
         @name    = name
         @version = `pkg-config --modversion #{name}`.chomp.strip
         @actions = Hash.new
         @variables = Hash.new
     end
 
-    ACTIONS = %w{cflags cflags-only-I cflags-only-other 
-                libs libs-only-L libs-only-l libs-only-other static}
+    ACTIONS = %w[cflags cflags-only-I cflags-only-other
+                 libs libs-only-L libs-only-l libs-only-other static].freeze
     ACTIONS.each do |action|
-        define_method(action.gsub(/-/, '_')) do 
+        define_method(action.tr('-', '_')) do
             @actions[action] ||= `pkg-config --#{action} #{name}`.chomp.strip
         end
     end
 
+    def respond_to_missing?(varname, _include_all)
+        varname =~ /^\w+$/
+    end
+
     def method_missing(varname, *args, &proc)
         if args.empty?
-            @variables[varname] ||= `pkg-config --variable=#{varname} #{name}`.chomp.strip
-        else
-            super(varname, *args, &proc)
+            unless (value = @variables[varname])
+                value = `pkg-config --variable=#{varname} #{name}`.chomp.strip
+                @variables[varname] = value
+            end
+            return value
         end
+        super
     end
 end
-

data/lib/autobuild/progress_display.rb

@@ -1,51 +1,106 @@
+require "concurrent/atomic/atomic_boolean"
+require "concurrent/array"
+
 module Autobuild
     # Management of the progress display
     class ProgressDisplay
         def initialize(io, color: ::Autobuild.method(:color))
             @io = io
-            #@cursor = Blank.new
             @cursor = TTY::Cursor
             @last_formatted_progress = []
-            @progress_messages = []
+            @progress_messages = Concurrent::Array.new
 
             @silent = false
             @color = color
-            @progress_enabled = true
             @display_lock = Mutex.new
+
+            @next_progress_display = Time.at(0)
+            @progress_mode = :single_line
+            @progress_period = 0.1
+
+            @message_queue = Queue.new
+            @forced_progress_display = Concurrent::AtomicBoolean.new(false)
         end
 
-        attr_writer :silent
+        def synchronize(&block)
+            result = @display_lock.synchronize(&block)
+            refresh_display
+            result
+        end
+
+        # Set the minimum time between two progress messages
+        #
+        # @see period
+        def progress_period=(period)
+            @progress_period = Float(period)
+        end
+
+        # Minimum time between two progress displays
+        #
+        # This does not affect normal messages
+        #
+        # @return [Float]
+        attr_reader :progress_period
+
+        # Valid progress modes
+        #
+        # @see progress_mode=
+        PROGRESS_MODES = %I[single_line newline off]
+
+        # Sets how progress messages will be displayed
+        #
+        # @param [String] the new mode. Can be either 'single_line', where a
+        #   progress message replaces the last one, 'newline' which displays
+        #   each on a new line or 'off' to disable progress messages altogether
+        def progress_mode=(mode)
+            mode = mode.to_sym
+            unless PROGRESS_MODES.include?(mode)
+                raise ArgumentError,
+                      "#{mode} is not a valid mode, expected one of "\
+                      "#{PROGRESS_MODES.join(", ")}"
+            end
+            @progress_mode = mode
+        end
+
+        # Return the current display mode
+        #
+        # @return [Symbol]
+        # @see mode=
+        attr_reader :progress_mode
 
         def silent?
             @silent
         end
 
+        def silent=(flag)
+            @silent = flag
+        end
+
         def silent
-            @silent, silent = true, @silent
+            silent = @silent
+            @silent = true
             yield
         ensure
             @silent = silent
         end
 
-        attr_writer :progress_enabled
+        # @deprecated use progress_mode= instead
+        def progress_enabled=(flag)
+            self.progress_mode = flag ? :single_line : :off
+        end
 
+        # Whether progress messages will be displayed at all
         def progress_enabled?
-            !@silent && @progress_enabled
+            !@silent && (@progress_mode != :off)
         end
 
         def message(message, *args, io: @io, force: false)
             return if silent? && !force
 
-            if args.last.respond_to?(:to_io)
-                io = args.pop
-            end
+            io = args.pop if args.last.respond_to?(:to_io)
+            @message_queue << [message, args, io]
 
-            @display_lock.synchronize do
-                io.print "#{@cursor.column(1)}#{@cursor.clear_screen_down}#{@color.call(message, *args)}\n"
-                io.flush if @io != io
-                display_progress
-                @io.flush
-            end
+            refresh_display
         end
 
         def progress_start(key, *args, done_message: nil)
@@ -54,13 +109,13 @@ def progress_start(key, *args, done_message: nil)
             formatted_message = @color.call(*args)
             @progress_messages << [key, formatted_message]
             if progress_enabled?
-                @display_lock.synchronize do
-                    display_progress
-                end
+                @forced_progress_display.make_true
             else
                 message "  #{formatted_message}"
             end
 
+            refresh_display
+
             if block_given?
                 begin
                     result = yield
@@ -74,76 +129,101 @@ def progress_start(key, *args, done_message: nil)
         end
 
         def progress(key, *args)
-            @display_lock.synchronize do
-                found = false
-                @progress_messages.map! do |msg_key, msg|
-                    if msg_key == key
-                        found = true
-                        [msg_key, @color.call(*args)]
-                    else
-                        [msg_key, msg]
-                    end
+            found = false
+            @progress_messages.map! do |msg_key, msg|
+                if msg_key == key
+                    found = true
+                    [msg_key, @color.call(*args)]
+                else
+                    [msg_key, msg]
                 end
-                @progress_messages << [key, @color.call(*args)] unless found
-                display_progress
             end
+            @progress_messages << [key, @color.call(*args)] unless found
+
+            refresh_display
         end
 
         def progress_done(key, display_last = true, message: nil)
-            changed = @display_lock.synchronize do
-                current_size = @progress_messages.size
-                @progress_messages.delete_if do |msg_key, msg|
-                    if msg_key == key
-                        if display_last && !message
-                            message = msg
-                        end
-                        true
-                    end
+            current_size = @progress_messages.size
+            @progress_messages.delete_if do |msg_key, msg|
+                if msg_key == key
+                    message = msg if display_last && !message
+                    true
                 end
-                current_size != @progress_messages.size
             end
+            changed = current_size != @progress_messages.size
 
             if changed
                 if message
                     message("  #{message}")
-                    # Note: message calls display_progress already
+                    # Note: message updates the display already
                 else
-                    @display_lock.synchronize do
-                        display_progress
-                    end
+                    refresh_display
                 end
                 true
             end
         end
 
+        def refresh_display
+            return unless @display_lock.try_lock
+
+            begin
+                refresh_display_under_lock
+            ensure
+                @display_lock.unlock
+            end
+        end
+
+        def refresh_display_under_lock
+            # Display queued messages
+            until @message_queue.empty?
+                message, args, io = @message_queue.pop
+                if @progress_mode == :single_line
+                    io.print @cursor.clear_screen_down
+                end
+                io.puts @color.call(message, *args)
+
+                io.flush if @io != io
+            end
+
+            # And re-display the progress
+            display_progress(consider_period: @forced_progress_display.false?)
+            @forced_progress_display.make_false
+            @io.flush
+        end
 
-        def display_progress
+        def display_progress(consider_period: true)
             return unless progress_enabled?
+            return if consider_period && (@next_progress_display > Time.now)
 
-            formatted = format_grouped_messages(@progress_messages.map(&:last), indent: "  ")
-            @io.print @cursor.clear_screen_down
-            @io.print formatted.join("\n")
-            if formatted.size > 1
-                @io.print "#{@cursor.up(formatted.size - 1)}#{@cursor.column(0)}"
+            formatted = format_grouped_messages(
+                @progress_messages.map(&:last),
+                indent: "  "
+            )
+            if @progress_mode == :newline
+                @io.print formatted.join("\n")
+                @io.print "\n"
             else
+                @io.print @cursor.clear_screen_down
+                @io.print formatted.join("\n")
+                @io.print @cursor.up(formatted.size - 1) if formatted.size > 1
                 @io.print @cursor.column(0)
             end
             @io.flush
+            @next_progress_display = Time.now + @progress_period
         end
 
         def find_common_prefix(msg, other_msg)
-            msg = msg.split(" ")
-            other_msg = other_msg.split(" ")
+            msg = msg.split(' ')
+            other_msg = other_msg.split(' ')
             msg.each_with_index do |token, idx|
                 if other_msg[idx] != token
                     prefix = msg[0..(idx - 1)].join(" ")
-                    if !prefix.empty?
-                        prefix << " "
-                    end
+                    prefix << ' ' unless prefix.empty?
                     return prefix
                 end
             end
-            return msg.join(" ")
+            msg.join(' ')
         end
 
         def group_messages(messages)
@@ -152,19 +232,22 @@ def group_messages(messages)
             groups = Array.new
             groups << ["", (0...messages.size)]
             messages.each_with_index do |msg, idx|
-                prefix, grouping = nil, false
+                prefix = nil
+                grouping = false
                 messages[(idx + 1)..-1].each_with_index do |other_msg, other_idx|
                     other_idx += idx + 1
                     prefix ||= find_common_prefix(msg, other_msg)
-                    break if !other_msg.start_with?(prefix)
+                    break unless other_msg.start_with?(prefix)
 
                     if grouping
                         break if prefix != groups.last[0]
+
                         groups.last[1] << other_idx
                     else
                         current_prefix, current_group = groups.last
-                        if prefix.size > current_prefix.size # create a new group from there
-                            groups.last[1] = (current_group.first..[idx-1,current_group.last].min)
+                        if prefix.size > current_prefix.size # create a new group
+                            group_end_index = [idx - 1, current_group.last].min
+                            groups.last[1] = (current_group.first..group_end_index)
                             groups << [prefix, [idx, other_idx]]
                             grouping = true
                         else break
@@ -179,13 +262,14 @@ def group_messages(messages)
             groups.map do |prefix, indexes|
                 indexes = indexes.to_a
                 next if indexes.empty?
+
                 range = (prefix.size)..-1
                 [prefix, indexes.map { |i| messages[i][range] }]
             end.compact
         end
 
-        def format_grouped_messages(messages, indent: "  ", width: TTY::Screen.width)
-            groups = group_messages(messages)
+        def format_grouped_messages(raw_messages, indent: "  ", width: TTY::Screen.width)
+            groups = group_messages(raw_messages)
             groups.each_with_object([]) do |(prefix, messages), lines|
                 if prefix.empty?
                     lines.concat(messages.map { |m| "#{indent}#{m.strip}" })
@@ -197,10 +281,11 @@ def format_grouped_messages(messages, indent: "  ", width: TTY::Screen.width)
                     msg = messages.shift.strip
                     margin = messages.empty? ? 1 : 2
                     if lines.last.size + margin + msg.size > width
-                        lines << "".dup
+                        lines.last << ","
+                        lines << +""
                         lines.last << indent << indent << msg
                     else
-                        lines.last << " " << msg
+                        lines.last << ", " << msg
                     end
                 end
                 lines.last << "," unless messages.empty?