autoproj 2.0.0.b5 → 2.0.0.b6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cfa3cb994d982b9d14f2a88e2b45959e42a12065
-  data.tar.gz: 041609d6fbe3a41b52e8d78992311bbfea212f71
+  metadata.gz: 7ac674274886714c8971929bc69953eb9a8554de
+  data.tar.gz: 1722b19143830e552aad26239225e742dd6bda78
 SHA512:
-  metadata.gz: 748d75cb0dfa737f899f84a6519e25b84125dee8dae9f1ddd9fb768f3c7100ed599ccb3c7fa59657ad78dc4e2186d9e575735ec9535cf8fd79900e106a8a7177
-  data.tar.gz: b221afdac5fbba3958b71aaec7e7c88bc1100ac5d58768d76bf98077e6a00e418786dbb6fa7d5ea7d90c118d1870625ad8e5f21b81b599392bdc6c71dcad4cd3
+  metadata.gz: 13148336716af711440e24212d70f83330ec82336ddb77cad4c32cb9d487a27f1180b523628a0dfd5812b45221bb1a805a679a55bcf0360cda969c2036f6d9a4
+  data.tar.gz: 6a487132bf1e90c569cb534e01706627c942d1bdf58b689786fc440c836fd2a3ecdf76d2c9cc85d58ee8e02dd8e7ac1c6b0718629d9090037f0e243437465fe2

data/Rakefile

@@ -5,15 +5,15 @@ task 'default'
 Utilrb::Rake.hoe do
     namespace 'dist' do
         Hoe.spec 'autoproj' do
-            self.developer "Rock Core Developers", "rock-dev@dfki.de"
+            self.developer "Sylvain Joyeux", "sylvain.joyeux@m4x.org"
 
             self.urls = ["http://rock-robotics.org/documentation/autoproj"]
             self.group_name = 'autobuild'
             self.summary = 'Easy installation and management of sets of software packages'
             self.description = "autoproj is a manager for sets of software packages. It allows the user to import and build packages from source, still using the underlying distribution's native package manager for software that is available on it."
-            self.email = "rock-dev@dfki.de"
+            self.email = "sylvain.joyeux@m4x.org"
 
-            self.spec_extras[:required_ruby_version] = ">= 1.9.2"
+            self.spec_extras[:required_ruby_version] = ">= 1.9.3"
 
             license 'BSD'
 
@@ -21,7 +21,8 @@ Utilrb::Rake.hoe do
                 ['autobuild',   '~> 1.9', '>= 1.9.0'] <<
                 ['utilrb', '~> 2.0', '>= 2.0.0'] <<
                 ['highline', '~> 1.7', '>= 1.7.0'] <<
-                ['thor', '~> 0.19.0', '>= 0.19.1']
+                ['thor', '~> 0.19.0', '>= 0.19.1'] <<
+                ['concurrent-ruby']
         end
     end
 end

data/bin/autoproj_bootstrap

@@ -385,6 +385,14 @@ module Autoproj
             set('import_log_enabled', !!value)
         end
 
+        def parallel_import_level
+            get('parallel_import_level', 10)
+        end
+
+        def parallel_import_level=(level)
+            set('parallel_import_level', level)
+        end
+
         def ruby_executable
             if path = get('ruby_executable', nil)
                 path

data/lib/autoproj.rb

@@ -1,5 +1,6 @@
 require "enumerator"
 require 'utilrb/module/attr_predicate'
+require 'concurrent'
 require 'autobuild'
 require 'autoproj/base'
 require 'autoproj/exceptions'

data/lib/autoproj/cli/base.rb

@@ -127,7 +127,8 @@ module Autoproj
                     verbose: false,
                     debug: false,
                     color: true,
-                    progress: true
+                    progress: true,
+                    parallel: nil
 
                 Autoproj.silent = options[:silent]
                 if options[:verbose]
@@ -145,6 +146,10 @@ module Autoproj
                 end
 
                 Autobuild.color = options[:color]
+                if level = options[:parallel]
+                    Autobuild.parallel_build_level = Integer(level)
+                    remaining[:parallel] = Integer(level)
+                end
 
                 Autobuild.progress_display_enabled = options[:progress]
                 return args, remaining

data/lib/autoproj/cli/build.rb

@@ -25,7 +25,9 @@ module Autoproj
                 Autobuild.ignore_errors = options[:keep_going]
 
                 command_line_selection, source_packages, osdep_packages =
-                    super(selected_packages, options.merge(checkout_only: true, apply_post_install: false))
+                    super(selected_packages, options.merge(checkout_only: true))
+
+                return if source_packages.empty?
 
                 # Disable all packages that are not selected
                 ws.manifest.each_autobuild_package do |pkg|
@@ -70,7 +72,6 @@ module Autoproj
                 Autobuild.do_build = true
                 ops.build_packages(source_packages)
                 Autobuild.apply(source_packages, "autoproj-build", ['install'])
-                ws.apply_post_install
             end
         end
     end

data/lib/autoproj/cli/inspection_tool.rb

@@ -12,7 +12,7 @@ module Autoproj
                 Autoproj.silent do
                     ws.setup
                     if mainline = options[:mainline]
-                        if mainline == 'true'
+                        if mainline == 'mainline' || mainline == 'true'
                             mainline = true
                         end
                     end
@@ -48,7 +48,6 @@ module Autoproj
                     source_packages, osdep_packages, resolved_selection =
                         resolve_selection(ws.manifest, packages, options)
                     ws.finalize_setup
-                    ws.apply_post_install
                     ws.export_installation_manifest
                     return source_packages, osdep_packages, resolved_selection, config_selected
                 end

data/lib/autoproj/cli/main.rb

@@ -93,6 +93,17 @@ module Autoproj
                 desc: 'controls whether the osdeps subsystem should filter up-to-date packages or not', default: true
             option :deps, default: true, type: :boolean,
                 desc: 'whether the package dependencies should be recursively updated (the default) or not'
+            option :reset, default: false, type: :boolean,
+                desc: "forcefully resets the repository to the state expected by autoproj's configuration",
+                long_desc: "The default is to update the repository if possible, and leave it alone otherwise. With --reset, autoproj update might come back to an older commit than the repository's current state"
+            option :force_reset, default: false, type: :boolean,
+                desc: "like --reset, but bypasses tests that ensure you won't lose data"
+            option :retry_count, default: nil, type: :numeric,
+                desc: "force the importer's retry count to this value"
+            option :parallel, aliases: :p, type: :numeric,
+                desc: 'maximum number of parallel jobs'
+            option :mainline, type: :string,
+                desc: "compare to the given baseline. if 'true', the comparison will ignore any override, otherwise it will take into account overrides only up to the given package set"
             def update(*packages)
                 run_autoproj_cli(:update, :Update, Hash[silent: false], *packages)
             end
@@ -112,6 +123,8 @@ module Autoproj
                 desc: 'controls whether missing osdeps should be installed. In rebuild mode, also controls whether the osdeps should be reinstalled or not (the default is to reinstall them)' 
             option :deps, type: :boolean,
                 desc: 'in force or rebuild modes, control whether the force/rebuild action should apply only on the packages given on the command line, or on their dependencies as well (the default is --no-deps)'
+            option :parallel, aliases: :p, type: :numeric,
+                desc: 'maximum number of parallel jobs'
             def build(*packages)
                 run_autoproj_cli(:build, :Build, Hash[silent: false], *packages)
             end
@@ -164,10 +177,19 @@ module Autoproj
             end
 
             desc 'versions [PACKAGES]', 'generate a version file for the given packages, or all packages if none are given'
-            option :package_sets, type: :boolean,
+            option :config, type: :boolean,
                 default: nil,
                 banner: '',
-                desc: 'controls whether the package sets should be versioned as well. This is the default if no packages are given on the command line or if the autoproj directory is'
+                desc: "controls whether the package sets should be versioned as well",
+                long_desc: <<-EOD
+This is the default if no packages are given on the command line, or if the
+autoproj main configuration directory is. Note that if --config but no packages
+are given, the packages will not be versioned. In other words,
+   autoproj versions # versions everything, configuration and packages
+   autoproj versions --config # versions only the configuration
+   autoproj versions autoproj/ # versions only the configuration
+   autoproj versions autoproj a/package # versions the configuration and the specified package(s)
+                EOD
             option :keep_going, aliases: :k, type: :boolean,
                 default: false,
                 banner: '',
@@ -223,32 +245,36 @@ module Autoproj
                 run_autoproj_cli(:switch_config, :SwitchConfig, Hash[], *args)
             end
 
-            desc 'query <query string>', 'searches for packages matching a query string'
+            desc 'query [QUERY]', 'searches for packages matching a query string. With no query string, matches all packages.'
             long_desc <<-EOD
   Finds packages that match query_string and displays information about them (one per line)
   By default, only the package name is displayed. It can be customized with the --format option
 
   QUERY KEYS
-    autobuild.name: the package name
-    autobuild.srcdir: the package source directory
-    autobuild.class.name: the package class
-    vcs.type: the VCS type (as used in the source.yml files)
-    vcs.url: the URL from the VCS. The exact semantic of it depends on the VCS type
+
+    autobuild.name: the package name,
+    autobuild.srcdir: the package source directory,
+    autobuild.class.name: the package class,
+    vcs.type: the VCS type (as used in the source.yml files),
+    vcs.url: the URL from the VCS,
     package_set.name: the name of the package set that defines the package
 
   FORMAT SPECIFICATION
 
   The format is a string in which special values can be expanded using a $VARNAME format. The following variables are accepted:
-    NAME: the package name
-    SRCDIR: the full path to the package source directory
+
+    NAME: the package name,
+
+    SRCDIR: the full path to the package source directory,
+
     PREFIX: the full path to the package installation directory
             EOD
             option :search_all, type: :boolean,
                 desc: 'search in all defined packages instead of only in those selected selected in the layout'
             option :format, type: :string,
                 desc: "customize what should be displayed. See FORMAT SPECIFICATION above"
-            def query(query_string)
-                run_autoproj_cli(:query, :Query, Hash[], query_string)
+            def query(query_string = nil)
+                run_autoproj_cli(:query, :Query, Hash[], *Array(query_string))
             end
         end
     end

data/lib/autoproj/cli/query.rb

@@ -6,32 +6,55 @@ module Autoproj
         class Query < InspectionTool
             def run(query_string, options = Hash.new)
                 initialize_and_load
-                finalize_setup([])
-                query = Autoproj::Query.parse_query(query_string.first)
-
-                packages =
-                    if options[:search_all]
-                        ws.manifest.packages.to_a
-                    else
-                        ws.manifest.all_selected_packages.map do |pkg_name|
-                            [pkg_name, ws.manifest.packages[pkg_name]]
-                        end
+                all_selected_packages, * =
+                    finalize_setup([], ignore_non_imported_packages: true)
+                all_selected_packages = all_selected_packages.to_set
+
+                query =
+                    if !query_string.empty?
+                        Autoproj::Query.parse_query(query_string.first)
                     end
 
-                matches = packages.map do |name, pkg_def|
-                    if priority = query.match(pkg_def)
-                        [priority, name]
+                if options[:search_all]
+                    packages = ws.manifest.packages.to_a
+                else
+                    packages = all_selected_packages.map do |pkg_name|
+                        [pkg_name, ws.manifest.find_package(pkg_name)]
+                    end
+                    packages += ws.manifest.all_selected_packages.map do |pkg_name|
+                        if !all_selected_packages.include?(pkg_name)
+                            [pkg_name, ws.manifest.find_package(pkg_name)]
+                        end
+                    end.compact
+                end
+
+                if options[:only_present]
+                    packages = packages.find_all do |_, pkg|
+                        File.directory?(pkg.autobuild.srcdir)
                     end
-                end.compact
+                end
+
+                if !query
+                    matches = packages.map { |name, _| [0, name] }
+                else
+                    matches = packages.map do |name, pkg_def|
+                        if priority = query.match(pkg_def)
+                            [priority, name]
+                        end
+                    end.compact
+                end
 
                 fields = Hash.new
                 matches = matches.sort
                 matches.each do |priority, name|
+                    pkg_def = ws.manifest.find_package(name)
                     pkg = ws.manifest.find_autobuild_package(name)
                     fields['SRCDIR'] = pkg.srcdir
                     fields['PREFIX'] = pkg.prefix
                     fields['NAME'] = name
                     fields['PRIORITY'] = priority
+                    fields['URL'] = (pkg_def.vcs.url if pkg_def.vcs)
+                    fields['PRESENT'] = File.directory?(pkg.srcdir)
 
                     value = Autoproj.expand(options[:format] || "$NAME", fields)
                     puts value

data/lib/autoproj/cli/status.rb

@@ -43,81 +43,99 @@ module Autoproj
                 display_status(packages, only_local: options[:only_local])
             end
 
+            PackageStatus = Struct.new :msg, :sync, :uncommitted, :local, :remote
+            def status_of_package(pkg, options = Hash.new)
+                package_status = PackageStatus.new(Array.new, false, false, false, false)
+                if !pkg.importer
+                    package_status.msg << Autoproj.color("  is a local-only package (no VCS)", :bold, :red)
+                elsif !pkg.importer.respond_to?(:status)
+                    package_status.msg << Autoproj.color("  the #{pkg.importer.class.name.gsub(/.*::/, '')} importer does not support status display", :bold, :red)
+                elsif !File.directory?(pkg.srcdir)
+                    package_status.msg << Autoproj.color("  is not imported yet", :magenta)
+                else
+                    begin status = pkg.importer.status(pkg, options[:only_local])
+                    rescue Interrupt
+                        raise
+                    rescue Exception => e
+                        package_status.msg << Autoproj.color("  failed to fetch status information (#{e})", :red)
+                        return package_status
+                    end
+
+                    if status.uncommitted_code
+                        package_status.msg << Autoproj.color("  contains uncommitted modifications", :red)
+                        package_status.uncommitted = true
+                    end
+
+                    case status.status
+                    when Autobuild::Importer::Status::UP_TO_DATE
+                        package_status.sync = true
+                    when Autobuild::Importer::Status::ADVANCED
+                        package_status.local = true
+                        package_status.msg << Autoproj.color("  local contains #{status.local_commits.size} commit that remote does not have:", :blue)
+                        status.local_commits.each do |line|
+                            package_status.msg << Autoproj.color("    #{line}", :blue)
+                        end
+                    when Autobuild::Importer::Status::SIMPLE_UPDATE
+                        package_status.remote = true
+                        package_status.msg << Autoproj.color("  remote contains #{status.remote_commits.size} commit that local does not have:", :magenta)
+                        status.remote_commits.each do |line|
+                            package_status.msg << Autoproj.color("    #{line}", :magenta)
+                        end
+                    when Autobuild::Importer::Status::NEEDS_MERGE
+                        package_status.local  = true
+                        package_status.remote = true
+                        package_status.msg << "  local and remote have diverged with respectively #{status.local_commits.size} and #{status.remote_commits.size} commits each"
+                        package_status.msg << Autoproj.color("  -- local commits --", :blue)
+                        status.local_commits.each do |line|
+                            package_status.msg << Autoproj.color("   #{line}", :blue)
+                        end
+                        package_status.msg << Autoproj.color("  -- remote commits --", :magenta)
+                        status.remote_commits.each do |line|
+                            package_status.msg << Autoproj.color("   #{line}", :magenta)
+                        end
+                    end
+                end
+                package_status
+            end
+
             StatusResult = Struct.new :uncommitted, :local, :remote
             def display_status(packages, options = Hash.new)
                 result = StatusResult.new
 
+                executor = Concurrent::FixedThreadPool.new(ws.config.parallel_import_level, max_length: 0)
+                interactive, noninteractive = packages.partition { |pkg| pkg.importer && pkg.importer.interactive? }
+                noninteractive = noninteractive.map do |pkg|
+                    [pkg, Concurrent::Future.execute(executor: executor) { status_of_package(pkg, only_local: options[:only_local]) }]
+                end
+
                 sync_packages = ""
-                packages.each do |pkg|
-                    lines = []
+                (noninteractive + interactive).each do |pkg, future|
+                    if future 
+                        if error = future.reason
+                            raise error
+                        end
+                        status = future.value
+                    else status = status_of_package(pkg, only_local: options[:only_local])
+                    end
 
-                    pkg_name = pkg.autoproj_name
+                    result.uncommitted ||= status.uncommitted
+                    result.local       ||= status.local
+                    result.remote      ||= status.remote
 
-                    if !pkg.importer
-                        lines << Autoproj.color("  is a local-only package (no VCS)", :bold, :red)
-                    elsif !pkg.importer.respond_to?(:status)
-                        lines << Autoproj.color("  the #{pkg.importer.class.name.gsub(/.*::/, '')} importer does not support status display", :bold, :red)
-                    elsif !File.directory?(pkg.srcdir)
-                        lines << Autoproj.color("  is not imported yet", :magenta)
-                    else
-                        status = begin pkg.importer.status(pkg, options[:only_local])
-                                 rescue Interrupt
-                                     raise
-                                 rescue Exception => e
-                                     lines << Autoproj.color("  failed to fetch status information (#{e})", :red)
-                                     nil
-                                 end
-
-                        if status
-                            if status.uncommitted_code
-                                lines << Autoproj.color("  contains uncommitted modifications", :red)
-                                result.uncommitted = true
-                            end
-
-                            case status.status
-                            when Autobuild::Importer::Status::UP_TO_DATE
-                                if !status.uncommitted_code
-                                    if sync_packages.size > 80
-                                        Autoproj.message "#{sync_packages},"
-                                        sync_packages = ""
-                                    end
-                                    msg = if sync_packages.empty?
-                                              pkg_name
-                                          else
-                                              ", #{pkg_name}"
-                                          end
-                                    STDERR.print msg
-                                    sync_packages = "#{sync_packages}#{msg}"
-                                    next
-                                else
-                                    lines << Autoproj.color("  local and remote are in sync", :green)
-                                end
-                            when Autobuild::Importer::Status::ADVANCED
-                                result.local = true
-                                lines << Autoproj.color("  local contains #{status.local_commits.size} commit that remote does not have:", :blue)
-                                status.local_commits.each do |line|
-                                    lines << Autoproj.color("    #{line}", :blue)
-                                end
-                            when Autobuild::Importer::Status::SIMPLE_UPDATE
-                                result.remote = true
-                                lines << Autoproj.color("  remote contains #{status.remote_commits.size} commit that local does not have:", :magenta)
-                                status.remote_commits.each do |line|
-                                    lines << Autoproj.color("    #{line}", :magenta)
-                                end
-                            when Autobuild::Importer::Status::NEEDS_MERGE
-                                result.local  = true
-                                result.remote = true
-                                lines << "  local and remote have diverged with respectively #{status.local_commits.size} and #{status.remote_commits.size} commits each"
-                                lines << Autoproj.color("  -- local commits --", :blue)
-                                status.local_commits.each do |line|
-                                    lines << Autoproj.color("   #{line}", :blue)
-                                end
-                                lines << Autoproj.color("  -- remote commits --", :magenta)
-                                status.remote_commits.each do |line|
-                                    lines << Autoproj.color("   #{line}", :magenta)
-                                end
-                            end
+                    pkg_name = pkg.autoproj_name
+                    if status.sync && status.msg.empty?
+                        if sync_packages.size > 80
+                            Autoproj.message "#{sync_packages},"
+                            sync_packages = ""
                         end
+                        msg = if sync_packages.empty?
+                                  pkg_name
+                              else
+                                  ", #{pkg_name}"
+                              end
+                        STDERR.print msg
+                        sync_packages = "#{sync_packages}#{msg}"
+                        next
                     end
 
                     if !sync_packages.empty?
@@ -127,20 +145,28 @@ module Autoproj
 
                     STDERR.print 
 
-                    if lines.size == 1
-                        Autoproj.message "#{pkg_name}: #{lines.first}"
+                    if status.msg.size == 1
+                        Autoproj.message "#{pkg_name}: #{status.msg.first}"
                     else
                         Autoproj.message "#{pkg_name}:"
-                        lines.each do |l|
+                        status.msg.each do |l|
                             Autoproj.message l
                         end
                     end
                 end
                 if !sync_packages.empty?
                     Autoproj.message("#{sync_packages}: #{Autoproj.color("local and remote are in sync", :green)}")
-                    sync_packages = ""
                 end
                 return result
+
+            rescue Interrupt
+                Autoproj.warn "Interrupted, waiting for pending jobs to finish"
+            rescue Exception => e
+                Autoproj.error "internal error: #{e}, waiting for pending jobs to finish"
+                raise
+            ensure
+                executor.shutdown
+                executor.wait_for_termination
             end
 
         end