autoproj 1.8.2.b17 → 1.8.2.b18

data/bin/autoproj

@@ -150,7 +150,7 @@ EOTEXT
 
     # If in verbose mode, or if we only update sources, list the sources
     if Autoproj.verbose || Autoproj::CmdLine.display_configuration?
-        Autoproj::CmdLine.display_configuration(manifest, selected_packages)
+        Autoproj::CmdLine.display_configuration(manifest, selected_packages.packages)
     end
     
     if Autoproj::CmdLine.bootstrap?

data/lib/autoproj/cmdline.rb

@@ -509,7 +509,7 @@ module Autoproj
                             send(handler, relative_to_root.to_s)
                         end
                         setup_package_directories(pkg)
-                        selected_packages << pkg.name
+                        selected_packages.select(sel, pkg.name)
                         break(true)
                     end
 
@@ -521,40 +521,38 @@ module Autoproj
                 Autoproj.message("autoproj: wrong package selection on command line, cannot find a match for #{nonresolved.to_a.join(", ")}", :red)
                 exit 1
             elsif Autoproj.verbose
-                Autoproj.message "will install #{selected_packages.to_a.join(", ")}"
+                Autoproj.message "will install #{selected_packages.packages.keys.to_a.sort.join(", ")}"
             end
             selected_packages
         end
 
-        def self.verify_package_availability(pkg_name, stack = Array.new)
-            if reason = Autoproj.manifest.exclusion_reason(pkg_name)
-                raise ConfigError.new, "#{pkg_name} is excluded from the build: #{reason}"
-            end
-            pkg = Autobuild::Package[pkg_name]
-            if !pkg
-                raise ConfigError.new, "#{pkg_name} does not seem to exist"
-            end
-            stack << pkg_name
-
-            # Verify that its dependencies are there, and add
-            # them to the selected_packages set so that they get
-            # imported as well
-            pkg.dependencies.each do |dep_name|
-                if stack.include?(dep_name)
-                    raise ConfigError.new, "circular dependency #{pkg.name} => #{dep_name}"
+        def self.mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
+            root = !reason
+            chain.unshift pkg_name
+            if root
+                reason = Autoproj.manifest.exclusion_reason(pkg_name)
+            else
+                if chain.size == 1
+                    Autoproj.manifest.add_exclusion(pkg_name, "its dependency #{reason}")
+                else
+                    Autoproj.manifest.add_exclusion(pkg_name, "#{reason} (dependency chain: #{chain.join(">")}")
                 end
+            end
 
-                begin
-                    verify_package_availability(dep_name, stack.dup)
-                rescue ConfigError => e
-                    raise e, "#{pkg_name} depends on #{dep_name}, but #{e.message}"
+            return if !revdeps.has_key?(pkg_name)
+            revdeps[pkg_name].each do |dep_name|
+                if !Autoproj.manifest.excluded?(dep_name)
+                    mark_exclusion_along_revdeps(dep_name, revdeps, chain.dup, reason)
                 end
             end
         end
 
-        def self.import_packages(selected_packages)
-            selected_packages = selected_packages.dup.
+        def self.import_packages(selection)
+            selected_packages = selection.packages.
                 map do |pkg_name|
+                    if Autoproj.manifest.excluded?(pkg_name)
+                        raise
+                    end
                     pkg = Autobuild::Package[pkg_name]
                     if !pkg
                         raise ConfigError.new, "selected package #{pkg_name} does not exist"
@@ -562,107 +560,92 @@ module Autoproj
                     pkg
                 end.to_set
 
-            # First, import all packages that are already there to make
-            # automatic dependency discovery possible
-            old_update_flag = Autobuild.do_update
-            begin
-                Autobuild.do_update = false
-                packages = Autobuild::Package.each.
-                    find_all { |pkg_name, pkg| File.directory?(pkg.srcdir) }.
-                    delete_if { |pkg_name, pkg| Autoproj.manifest.excluded?(pkg_name) || Autoproj.manifest.ignored?(pkg_name) }
-
-                packages.each do |_, pkg|
-                    pkg.isolate_errors do
-                        pkg.import
-                    end
-                end
-
-            ensure
-                Autobuild.do_update = old_update_flag
-            end
-
-            known_available_packages = Set.new
+            # The set of all packages that are currently selected by +selection+
             all_enabled_packages = Set.new
+            # The reverse dependencies for the package tree. It is discovered as
+            # we go on with the import
+            #
+            # It only contains strong dependencies. Optional dependencies are
+            # not included, as we will use this only to take into account
+            # package exclusion (and that does not affect optional dependencies)
+            reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }
 
-            package_queue = selected_packages.dup
+            package_queue = selected_packages.to_a.sort_by(&:name)
             while !package_queue.empty?
-                current_packages, package_queue = package_queue, Set.new
-                current_packages = current_packages.sort_by(&:name)
-
-                current_packages.
-                    delete_if { |pkg| all_enabled_packages.include?(pkg.name) }.
-                    delete_if { |pkg| Autoproj.manifest.ignored?(pkg.name) }
-                all_enabled_packages |= current_packages.map(&:name).to_set
-
-                # Recursively check that no package in the selection depend on
-                # excluded packages
-                current_packages.each do |pkg|
-                    verify_package_availability(pkg.name)
-                end
-
-                # We import first so that all packages can export the
-                # additional targets they provide.
-                current_packages.each do |pkg|
-                    # If the package has no importer, the source directory must
-                    # be there
-                    if !pkg.importer && !File.directory?(pkg.srcdir)
-                        raise ConfigError.new, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
+                pkg = package_queue.shift
+                # Remove packages that have already been processed
+                next if all_enabled_packages.include?(pkg.name)
+                all_enabled_packages << pkg.name
+
+                # If the package has no importer, the source directory must
+                # be there
+                if !pkg.importer && !File.directory?(pkg.srcdir)
+                    raise ConfigError.new, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
+                end
+
+                ## COMPLETELY BYPASS RAKE HERE
+                # The reason is that the ordering of import/prepare between
+                # packages is not important BUT the ordering of import vs.
+                # prepare in one package IS important: prepare is the method
+                # that takes into account dependencies.
+                pkg.import
+                Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
+                manifest.load_package_manifest(pkg.name)
+
+                # The package setup mechanisms might have added an exclusion
+                # on this package. Handle this.
+                if Autoproj.manifest.excluded?(pkg.name)
+                    mark_exclusion_along_revdeps(pkg.name, reverse_dependencies)
+                    # Run a filter now, to have errors as early as possible
+                    selection.filter_excluded_and_ignored_packages(Autoproj.manifest)
+                    all_enabled_packages.delete(pkg.name)
+                    # Delete this package from the current_packages set
+                    true
+                end
+
+                Autoproj.each_post_import_block(pkg) do |block|
+                    block.call(pkg)
+                end
+
+                pkg.prepare
+                Rake::Task["#{pkg.name}-prepare"].instance_variable_set(:@already_invoked, true)
+
+                # Verify that its dependencies are there, and add
+                # them to the selected_packages set so that they get
+                # imported as well
+                new_packages = []
+                pkg.dependencies.each do |dep_name|
+                    reverse_dependencies[dep_name] << pkg.name
+                    new_packages << Autobuild::Package[dep_name]
+                end
+                pkg_opt_deps, _ = pkg.partition_optional_dependencies
+                pkg_opt_deps.each do |dep_name|
+                    new_packages << Autobuild::Package[dep_name]
+                end
+
+                new_packages.delete_if do |pkg|
+                    if Autoproj.manifest.excluded?(pkg.name)
+                        mark_exclusion_along_revdeps(pkg.name, reverse_dependencies)
+                        true
                     end
-
-                    ## COMPLETELY BYPASS RAKE HERE
-                    # The reason is that the ordering of import/prepare between
-                    # packages is not important BUT the ordering of import vs.
-                    # prepare in one package IS important: prepare is the method
-                    # that takes into account dependencies.
-                    pkg.import
-                    Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
-                    manifest.load_package_manifest(pkg.name)
-                    pkg.resolve_optional_dependencies
-                    verify_package_availability(pkg.name)
                 end
+                package_queue.concat(new_packages.sort_by(&:name))
 
-                current_packages.each do |pkg|
-                    verify_package_availability(pkg.name)
-                    Autoproj.each_post_import_block(pkg) do |block|
-                        block.call(pkg)
-                    end
-
-                    pkg.prepare
-                    Rake::Task["#{pkg.name}-prepare"].instance_variable_set(:@already_invoked, true)
-
-                    # Verify that its dependencies are there, and add
-                    # them to the selected_packages set so that they get
-                    # imported as well
-                    pkg.dependencies.each do |dep_name|
-                        package_queue << Autobuild::Package[dep_name]
-                    end
-                end
+                # Verify that everything is still OK with the new
+                # exclusions/ignores
+                selection.filter_excluded_and_ignored_packages(Autoproj.manifest)
             end
 
-            begin
-                Autobuild.do_update = false
-                packages = Autobuild::Package.each.
-                    find_all { |pkg_name, pkg| File.directory?(pkg.srcdir) }.
-                    delete_if { |pkg_name, pkg| all_enabled_packages.include?(pkg_name) || Autoproj.manifest.excluded?(pkg_name) || Autoproj.manifest.ignored?(pkg_name) }
-
-                packages.each do |_, pkg|
-                    pkg.isolate_errors do
-                        manifest.load_package_manifest(pkg.name)
-                        pkg.prepare
-
-                        Autoproj.each_post_import_block(pkg) do |block|
-                            block.call(pkg)
-                        end
-                    end
-                end
-
-            ensure
-                Autobuild.do_update = old_update_flag
+            # We finally resolve optional dependencies at the very end, as the
+            # list of exclusions may have changed
+            all_enabled_packages.each do |pkg_name|
+                Autobuild::Package[pkg_name].resolve_optional_dependencies
             end
 
             if Autoproj.verbose
                 Autoproj.message "autoproj: finished importing packages"
             end
+
             if Autoproj::CmdLine.list_newest?
                 fields = []
                 Rake::Task.tasks.each do |task|

data/lib/autoproj/manifest.rb

@@ -1144,11 +1144,11 @@ module Autoproj
         # tells why. Otherwise, returns nil
         #
         # Packages can either be excluded because their name is listed in the
-        # excluded_packages section of the manifest, or because they are
+        # exclude_packages section of the manifest, or because they are
         # disabled on this particular operating system.
         def exclusion_reason(package_name)
             if manifest_exclusions.any? { |l| Regexp.new(l) =~ package_name }
-                "#{package_name} is listed in the excluded_packages section of the manifest"
+                "#{package_name} is listed in the exclude_packages section of the manifest"
             else
                 automatic_exclusions[package_name]
             end
@@ -1805,21 +1805,19 @@ module Autoproj
         #
         # If recursive is false, yields only the packages at this level.
         # Otherwise, return all packages.
-        def layout_packages(result, validate)
+        def layout_packages(validate)
+            result = PackageSelection.new
             Autoproj.in_file(self.file) do
                 normalized_layout.each_key do |pkg_or_set|
-                    begin
-                        resolve_package_set(pkg_or_set).each do |pkg_name|
-                            if !excluded?(pkg_name) && !ignored?(pkg_name)
-                                result << pkg_name
-                            end
-                            Autobuild::Package[pkg_name].all_dependencies(result)
-                        end
-                    rescue ConfigError
-                        raise if validate
-                    end
+                    result.select(pkg_or_set, resolve_package_set(pkg_or_set))
                 end
             end
+            
+            begin
+                result.filter_excluded_and_ignored_packages(self)
+            rescue ConfigError
+                raise if validate
+            end
             result
         end
 
@@ -1898,15 +1896,18 @@ module Autoproj
         # Returns the set of packages that should be built if the user does not
         # specify any on the command line
         def default_packages(validate = true)
-            names = if layout = data['layout']
-                        layout_packages(Set.new, validate)
-                    else
-                        # No layout, all packages are selected
-                        all_packages
-                    end
-
-            names.delete_if { |pkg_name| excluded?(pkg_name) || ignored?(pkg_name) }
-            names.to_set
+            if layout = data['layout']
+                return layout_packages(validate)
+            else
+                result = PackageSelection.new
+                # No layout, all packages are selected
+                names = all_packages
+                names.delete_if { |pkg_name| excluded?(pkg_name) || ignored?(pkg_name) }
+                names.each do |pkg_name|
+                    result.select(pkg_name, pkg_name)
+                end
+                result
+            end
         end
 
         def normalized_layout(result = Hash.new, layout_level = '/', layout_data = (data['layout'] || Hash.new))
@@ -2070,6 +2071,101 @@ module Autoproj
             @osdeps_overrides.delete(osdeps_name.to_s)
         end
 
+        # Class holding information about which packages have been selected, and
+        # why. It is used to decide whether some non-availability of packages
+        # are errors or simply warnings (i.e. if the user really wants a given
+        # package, or merely might be adding it by accident)
+        class PackageSelection
+            # The set of matches, i.e. a mapping from a user-provided string to
+            # the set of packages it selected
+            attr_reader :matches
+            # The set of selected packages, as a hash of the package name to the
+            # set of user-provided strings that caused that package to be
+            # selected
+            attr_reader :selection
+
+            # The set of packages that have been selected
+            def packages
+                selection.keys
+            end
+
+            def include?(pkg_name)
+                selection.has_key?(pkg_name)
+            end
+
+            def initialize
+                @selection = Hash.new { |h, k| h[k] = Set.new }
+                @matches = Hash.new { |h, k| h[k] = Set.new }
+            end
+
+            def select(sel, packages)
+                if !packages.respond_to?(:each)
+                    matches[sel] << packages
+                    selection[packages] << sel
+                else
+                    matches[sel] |= packages.to_set
+                    packages.each do |pkg_name|
+                        selection[pkg_name] << sel
+                    end
+                end
+            end
+
+            def initialize_copy(old)
+                old.selection.each do |pkg_name, set|
+                    @selection[pkg_name] = set.dup
+                end
+                old.matches.each do |sel, set|
+                    @matches[sel] = set.dup
+                end
+            end
+
+            # Remove packages that are explicitely excluded and/or ignored
+            #
+            # Raise an error if an explicit selection expands only to an
+            # excluded package, and display a warning for ignored packages
+            def filter_excluded_and_ignored_packages(manifest)
+                matches.each do |sel, expansion|
+                    excluded, other = expansion.partition { |pkg_name| manifest.excluded?(pkg_name) }
+                    ignored,  ok    = other.partition { |pkg_name| manifest.ignored?(pkg_name) }
+
+                    if ok.empty? && ignored.empty?
+                        exclusions = excluded.map do |pkg_name|
+                            [pkg_name, manifest.exclusion_reason(pkg_name)]
+                        end
+                        if exclusions.size == 1
+                            reason = exclusions[0][1]
+                            if sel == exclusions[0][0]
+                                raise ConfigError.new, "#{sel} is excluded from the build: #{reason}"
+                            else
+                                raise ConfigError.new, "#{sel} expands to #{exclusions.map(&:first).join(", ")}, which is excluded from the build: #{reason}"
+                            end
+                        else
+                            raise ConfigError.new, "#{sel} expands to #{exclusions.map(&:first).join(", ")}, and all these packages are excluded from the build:\n  #{exclusions.map { |name, reason| "#{name}: #{reason}" }.join("\n  ")}"
+                        end
+                    elsif !ignored.empty?
+                        ignored.each do |pkg_name|
+                            Autoproj.warn "#{pkg_name} was selected for #{sel}, but is explicitely ignored in the manifest"
+                        end
+                    end
+
+                    excluded = excluded.to_set
+                    expansion.delete_if do |pkg_name|
+                        excluded.include?(pkg_name)
+                    end
+                end
+
+                selection.keys.sort.each do |pkg_name|
+                    if manifest.excluded?(pkg_name)
+                        Autoproj.warn "#{pkg_name} was selected for #{selection[pkg_name].to_a.sort.join(", ")}, but it is excluded from the build: #{Autoproj.manifest.exclusion_reason(pkg_name)}"
+                        selection.delete(pkg_name)
+                    elsif manifest.ignored?(pkg_name)
+                        Autoproj.warn "#{pkg_name} was selected for #{selection[pkg_name].to_a.sort.join(", ")}, but it is ignored in this build"
+                        selection.delete(pkg_name)
+                    end
+                end
+            end
+        end
+
         # Package selection can be done in three ways:
         #  * as a subdirectory in the layout
         #  * as a on-disk directory
@@ -2079,14 +2175,10 @@ module Autoproj
         def expand_package_selection(selection)
             base_dir = Autoproj.root_dir
 
-            # The expanded selection
-            expanded_packages = Set.new
+            result = PackageSelection.new
             # All the packages that are available on this installation
             all_layout_packages = self.all_selected_packages
 
-            # A selection to packages map that represents the matches found
-            matches = Hash.new { |h, k| h[k] = Set.new }
-
             # First, remove packages that are directly referenced by name or by
             # package set names
             selection.each do |sel|
@@ -2096,16 +2188,14 @@ module Autoproj
                     find_all { |pkg_name| pkg_name =~ match_pkg_name }.
                     to_set
                 if !packages.empty?
-                    matches[sel] = packages
-                    expanded_packages |= packages
+                    result.select(sel, packages)
                 end
 
                 each_metapackage do |pkg|
                     if pkg.name =~ match_pkg_name
                         packages = resolve_package_set(pkg.name).to_set
                         packages = (packages & all_layout_packages)
-                        matches[sel] |= packages
-                        expanded_packages |= packages
+                        result.select(sel, packages)
                     end
                 end
             end
@@ -2125,37 +2215,13 @@ module Autoproj
                             end
                         end
 
-                        matches[sel] << pkg_name
-                        expanded_packages << pkg_name
-                    end
-                end
-            end
-
-            # Remove packages that are explicitely excluded and/or ignored
-            #
-            # Raise an error if an explicit selection expands only to an
-            # excluded package, and display a warning for ignored packages
-            matches.each do |sel, expansion|
-                next if expansion.empty?
-                excluded, other = expansion.partition { |pkg_name| excluded?(pkg_name) }
-                ignored,  ok    = other.partition { |pkg_name| ignored?(pkg_name) }
-
-                if ok.empty? && ignored.empty?
-                    packages = excluded.map do |pkg_name|
-                        [pkg_name, Autoproj.manifest.exclusion_reason(pkg_name)]
-                    end
-                    raise ConfigError.new, "selection #{sel} expands to #{packages.map(&:first).join(", ")} which are excluded from the build:\n  #{packages.map { |name, reason| "#{name}: #{reason}" }.join("\n  ")}"
-                elsif !ignored.empty?
-                    ignored.each do |pkg_name|
-                        Autoproj.warn "#{pkg_name} was selected for #{sel}, but is explicitely ignored in the manifest"
+                        result.select(sel, pkg_name)
                     end
                 end
             end
 
-            expanded_packages.delete_if do |pkg_name|
-                excluded?(pkg_name) || ignored?(pkg_name)
-            end
-            return expanded_packages.to_set, (selection - matches.keys)
+            result.filter_excluded_and_ignored_packages(self)
+            return result, (selection - matches.keys)
         end
 
         attr_reader :moved_packages

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "1.8.2.b17"
+    VERSION = "1.8.2.b18"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: autoproj
 version: !ruby/object:Gem::Version
-  version: 1.8.2.b17
+  version: 1.8.2.b18
   prerelease: 6
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: autobuild
-  requirement: &16486980 !ruby/object:Gem::Requirement
+  requirement: &12848100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0.rc1
   type: :runtime
   prerelease: false
-  version_requirements: *16486980
+  version_requirements: *12848100
 - !ruby/object:Gem::Dependency
   name: utilrb
-  requirement: &16484260 !ruby/object:Gem::Requirement
+  requirement: &12847280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *16484260
+  version_requirements: *12847280
 - !ruby/object:Gem::Dependency
   name: highline
-  requirement: &16482140 !ruby/object:Gem::Requirement
+  requirement: &12846560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 1.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *16482140
+  version_requirements: *12846560
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &16480960 !ruby/object:Gem::Requirement
+  requirement: &12845640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *16480960
+  version_requirements: *12845640
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &16480180 !ruby/object:Gem::Requirement
+  requirement: &12868200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '3.1'
   type: :development
   prerelease: false
-  version_requirements: *16480180
+  version_requirements: *12868200
 description: autoproj is a manager for sets of software packages. It allows the user
   to import and build packages from source, still using the underlying distribution's
   native package manager for software that is available on it.