autoproj 1.6.2 → 1.7.0.b1

data/History.txt

@@ -1,3 +1,24 @@
+= Version 1.7.0
+ * 1.6.2 should have been 1.7.0. Bump it now.
+ * introduced cross-package-set imports. A package set can now tell autoproj to
+   install another one in a recursive manner (see user's guide)
+ * added optional dependencies, which will not be considered if the
+   depended-upon package is either not existing or is explicitely excluded from
+   the build. It is specified in Ruby with
+       pkg.optional_dependency pkg_name
+   or in the package's manifest.xml files by setting the "optional" attribute to 1:
+       <depend package="pkg_name" optional="1" />
+
+ * added the #package toplevel method to get a package handle, i.e.
+       package('orocos/rtt')
+   is equivalent to the former
+       Autobuild::Package['orocos/rtt']
+ * added the --randomize-layout option which makes autoproj build each package
+   in a separate non-trivial subfolder, and install it in a separate folder as
+   well. This is meant to test the definition and handling of dependencies
+   between the different packages.
+ * added the autolocate tool, which returns the full path to a certain package
+   
 = Version 1.6.2
  * overhauled osdeps fine-grained configuration. See
    http://orocos.org/dfki/autoproj/osdeps.html
@@ -10,7 +31,7 @@
    errors are properly presented at the end of the run.
  * when boostrapping, check that the current directory is empty and warn the
    user if it is not.
- * a few minor bugfixes
+ * a few more minor bugfixes
 
 = Version 1.6.1
  * fix a limitation of giving directories on the command line. Before, only

data/Rakefile

@@ -14,7 +14,7 @@ begin
             self.changes     = paragraphs_of('History.txt', 0..1).join("\n\n")
 
             extra_deps << 
-                ['autobuild',   '>= 1.5.23'] <<
+                ['autobuild',   '>= 1.5.24'] <<
                 ['rmail',   '>= 1.0.0'] <<
                 ['utilrb', '>= 1.3.3'] <<
                 ['nokogiri', '>= 1.3.3'] <<

data/bin/autoproj

@@ -1,5 +1,12 @@
 #! /usr/bin/env ruby
 
+if !ARGV.grep("--env").empty?
+    require 'autoproj/system'
+    puts File.join(Autoproj.root_dir, "env.sh")
+    exit
+end
+
+
 if RUBY_VERSION < "1.8.7"
     STDERR.puts "autoproj requires Ruby >= 1.8.7"
     exit 1
@@ -103,9 +110,13 @@ EOTEXT
         Autobuild.do_update = Autoproj.auto_update?
     end
 
+    selected_packages = Autoproj::CmdLine.resolve_user_selection(selected_packages)
+    Autoproj.manifest.explicit_selection = selected_packages
+    Autoproj::CmdLine.initial_package_setup
+
     # If in verbose mode, or if we only update sources, list the sources
     if Autoproj.verbose || Autoproj::CmdLine.display_configuration?
-        Autoproj::CmdLine.display_sources(manifest)
+        Autoproj::CmdLine.display_configuration(manifest, selected_packages)
     end
     
     if Autoproj::CmdLine.bootstrap?
@@ -136,10 +147,6 @@ EOTEXT
         exit(0)
     end
 
-    selected_packages = Autoproj::CmdLine.resolve_user_selection(selected_packages)
-    Autoproj.manifest.explicit_selection = selected_packages
-    Autoproj::CmdLine.initial_package_setup
-
     if Autoproj::CmdLine.only_status?
         all_enabled_packages = Autoproj::CmdLine.import_packages(selected_packages)
         Autoproj::CmdLine.status(all_enabled_packages)
@@ -180,7 +187,13 @@ EOTEXT
     all_enabled_packages = Autoproj::CmdLine.import_packages(selected_packages)
 
     if Autoproj::CmdLine.update_os_dependencies? && !all_enabled_packages.empty?
-        manifest.install_os_dependencies(all_enabled_packages)
+        begin
+            update_mode = Autobuild.do_update
+            Autobuild.do_update ||= Autoproj::CmdLine.osdeps?
+            manifest.install_os_dependencies(all_enabled_packages)
+        ensure
+            Autobuild.do_update = update_mode
+        end
     end
 
     if all_enabled_packages.empty?

data/doc/guide/src/autoproj_bootstrap

@@ -106,7 +106,7 @@ module Autoproj
                 data = YAML.load(File.read(file)) || Hash.new
                 verify_definitions(data)
             rescue ArgumentError => e
-                raise ConfigError, "error in #{file}: #{e.message}"
+                raise ConfigError.new, "error in #{file}: #{e.message}"
             end
 
             OSDependencies.new(data, file)
@@ -243,8 +243,9 @@ module Autoproj
                 return @operating_system
             elsif Autoproj.has_config_key?('operating_system')
                 os = Autoproj.user_config('operating_system')
-                if !os.respond_to?(:to_ary) # upgrade from previous format
-                    @operating_system = nil
+                if os.respond_to?(:to_ary) # upgrade from previous format
+                    @operating_system = os
+                    return os
                 end
             end
 
@@ -434,7 +435,7 @@ fi
             elsif data.respond_to?(:to_str)
                 return [PACKAGES, [data.to_str]]
             else
-                raise ConfigError, "invalid package specificiation #{data} in #{source_of(name)}"
+                raise ConfigError.new, "invalid package specificiation #{data} in #{source_of(name)}"
             end
         end
 
@@ -449,11 +450,11 @@ fi
             dependencies.each do |name|
                 result = resolve_package(name)
                 if result == NO_PACKAGE
-                    raise ConfigError, "there is no osdeps definition for #{name}"
+                    raise ConfigError.new, "there is no osdeps definition for #{name}"
                 elsif result == WRONG_OS
-                    raise ConfigError, "there is an osdeps definition for #{name}, but not for this operating system"
+                    raise ConfigError.new, "there is an osdeps definition for #{name}, but not for this operating system"
                 elsif result == WRONG_OS_VERSION
-                    raise ConfigError, "there is an osdeps definition for #{name}, but not for this particular operating system version"
+                    raise ConfigError.new, "there is an osdeps definition for #{name}, but not for this particular operating system version"
                 elsif result == IGNORE
                     next
                 elsif result[0] == PACKAGES
@@ -462,7 +463,7 @@ fi
             end
 
             if !os_names.any? { |os_name| OS_AUTO_PACKAGE_INSTALL.has_key?(os_name) }
-                raise ConfigError, "I don't know how to install packages on #{os_names.first}"
+                raise ConfigError.new, "I don't know how to install packages on #{os_names.first}"
             end
 
             return os_packages
@@ -545,7 +546,7 @@ fi
                         # This is *not* handled later, as is the absence of a
                         # package definition. The reason is that it is a bad
                         # configuration file, and should be fixed by the user
-                        raise ConfigError, "unknown OS-independent package management type #{pkg_def} for #{name}"
+                        raise ConfigError.new, "unknown OS-independent package management type #{pkg_def} for #{name}"
                     end
                 else
                     pkg_def.delete_if do |distrib_name, defs|
@@ -609,7 +610,7 @@ fi
                     installed_version = installed.map(&:version).max
                     available_version = available.map { |(name, v), source| v }.max
                     if !available_version
-                        raise ConfigError, "cannot find any gem with the name '#{name}'"
+                        raise ConfigError.new, "cannot find any gem with the name '#{name}'"
                     end
                     needs_update = (available_version > installed_version)
                     !needs_update
@@ -950,7 +951,7 @@ module Autoproj
             @name, @type, @options = name.to_str, type.to_str, options.to_hash
             @validator = validator.to_proc if validator
             if !BuildOption.respond_to?("validate_#{type}")
-                raise ConfigError, "invalid option type #{type}"
+                raise ConfigError.new, "invalid option type #{type}"
             end
         end
 
@@ -1023,9 +1024,12 @@ module Autoproj
             if possible_values = options[:possible_values]
                 if options[:lowercase]
                     value = value.downcase
+                elsif options[:uppercase]
+                    value = value.upcase
                 end
+
                 if !possible_values.include?(value)
-                    raise InputError, "invalid value '#{value}', accepted values are '#{possible_values.join(", ")}'"
+                    raise InputError, "invalid value '#{value}', accepted values are '#{possible_values.join("', '")}' (without the quotes)"
                 end
             end
             value
@@ -1087,7 +1091,7 @@ module Autoproj
             @user_config[option_name] = [value, true]
             value
         else
-            raise ConfigError, "undeclared option '#{option_name}'"
+            raise ConfigError.new, "undeclared option '#{option_name}'"
         end
     end
 
@@ -1315,85 +1319,85 @@ end
 
 DEFS = <<EODEFS
 --- 
+svn: 
+  arch: subversion
+  gentoo: dev-util/subversion
+  debian,ubuntu: subversion
+autobuild: gem
+zlib: 
+  debian,ubuntu: zlib1g-dev
+libxml2: 
+  arch: libxml2
+  gentoo: dev-libs/libxml2
+  debian,ubuntu: libxml2-dev
 none: ignore
+autotools: 
+  arch: automake autoconf
+  gentoo: 
+  - sys-devel/automake:1.9
+  - sys-devel/autoconf
+  debian,ubuntu: 
+  - automake1.9
+  - autoconf
+autoproj: gem
+archive: 
+  arch: 
+  - tar
+  - unzip
+  gentoo: 
+  - app-arch/tar
+  - app-arch/unzip
+  debian,ubuntu: 
+  - tar
+  - unzip
+lsb_release: 
+  arch: 
+  gentoo: sys-apps/lsb-release
+  debian,ubuntu: lsb-release
 ruby18: 
+  gentoo: 
+  - dev-lang/ruby:1.8
   debian,ubuntu: 
   - ruby1.8-dev
   - ruby1.8
   - rubygems1.8
   - ri1.8
   - libopenssl-ruby1.8
-  gentoo: 
-  - dev-lang/ruby:1.8
 ruby19: 
   debian: 
   - ruby1.9.1
   - ruby1.9.1-dev
   - rubygems1.9.1
+  arch: 
+  - ruby
+  gentoo: 
+  - dev-lang/ruby:1.9
   ubuntu: 
   - ruby1.9.1
   - ruby1.9.1-dev
   - rubygems1.9.1
   - ri1.9.1
   - libopenssl-ruby1.9.1
-  gentoo: 
-  - dev-lang/ruby:1.9
-  arch: 
-  - ruby
-rdoc: gem
-build-essential: 
-  debian,ubuntu: build-essential
-  gentoo: 
-  arch: 
-libxml2: 
-  debian,ubuntu: libxml2-dev
-  gentoo: dev-libs/libxml2
-  arch: libxml2
-libxslt: 
-  debian,ubuntu: libxslt1-dev
-  gentoo: dev-libs/libxslt
-  arch: libxslt
-zlib: 
-  debian,ubuntu: zlib1g-dev
-autobuild: gem
-autoproj: gem
 git: 
   debian: 
     lenny: git
     default: git-core
-  ubuntu: git-core
-  gentoo: dev-vcs/git
   arch: git
-svn: 
-  debian,ubuntu: subversion
-  gentoo: dev-util/subversion
-  arch: subversion
+  gentoo: dev-vcs/git
+  ubuntu: git-core
 cmake: 
-  debian,ubuntu: cmake
-  gentoo: dev-util/cmake
   arch: cmake
-autotools: 
-  debian,ubuntu: 
-  - automake1.9
-  - autoconf
-  gentoo: 
-  - sys-devel/automake:1.9
-  - sys-devel/autoconf
-  arch: automake autoconf
-lsb_release: 
-  debian,ubuntu: lsb-release
-  gentoo: sys-apps/lsb-release
+  gentoo: dev-util/cmake
+  debian,ubuntu: cmake
+build-essential: 
   arch: 
-archive: 
-  debian,ubuntu: 
-  - tar
-  - unzip
   gentoo: 
-  - app-arch/tar
-  - app-arch/unzip
-  arch: 
-  - tar
-  - unzip
+  debian,ubuntu: build-essential
+libxslt: 
+  arch: libxslt
+  gentoo: dev-libs/libxslt
+  debian,ubuntu: libxslt1-dev
+rdoc: gem
 
 EODEFS
 

data/lib/autoproj/autobuild.rb

@@ -74,6 +74,27 @@ module Autobuild
             depends_on(name)
         end
 
+        def optional_dependency(name)
+            optional_dependencies << name
+        end
+
+        def resolve_optional_dependencies
+            optional_dependencies.each do |name|
+                if Autobuild::Package[name] && Autoproj.manifest.package_enabled?(name)
+                    if Autoproj.verbose
+                        STDERR.puts "adding optional dependency #{self.name} => #{name}"
+                    end
+                    depends_on(name)
+                elsif Autoproj.verbose
+                    STDERR.puts "NOT adding optional dependency #{self.name} => #{name}"
+                end
+            end
+        end
+
+        def optional_dependencies
+            @optional_dependencies ||= Set.new
+        end
+
         def os_packages
             @os_packages ||= Set.new
         end
@@ -133,9 +154,9 @@ module Autoproj
             end
 
             if source.local?
-                raise ConfigError, "#{path}:#{line_number} #{error.message}", error.backtrace
+                raise ConfigError.new(path), "#{path}:#{line_number} #{error.message}", error.backtrace
             else
-                raise ConfigError, "#{File.basename(path)}(source=#{source.name}):#{line_number} #{error.message}", error.backtrace
+                raise ConfigError.new(path), "#{File.basename(path)}(source=#{source.name}):#{line_number} #{error.message}", error.backtrace
             end
         else
             raise error
@@ -443,3 +464,18 @@ class Autobuild::ArchiveImporter
     end
 end
 
+def package(name)
+    Autobuild::Package[name]
+end
+
+# Returns true if +name+ is a valid package and is included in the build
+def package_enabled?(name)
+    Autoproj.manifest.package_enabled?(name, false)
+end
+
+# If used in init.rb, allows to disable automatic imports from specific package
+# sets
+def disable_imports_from(name)
+    Autoproj.manifest.disable_imports_from(name)
+end
+

data/lib/autoproj/base.rb

@@ -1,6 +1,28 @@
 module Autoproj
-    class ConfigError < RuntimeError; end
+    class ConfigError < RuntimeError
+        attr_accessor :file
+        def initialize(file = nil)
+            super
+            @file = file
+        end
+    end
     class InternalError < RuntimeError; end
+
+    # Yields, and if the given block raises a ConfigError with no file assigned,
+    # add that file to both the object and the exception message
+    def self.in_file(file, exception_t = ConfigError)
+        yield
+
+    rescue exception_t => e
+        if exception_t != ConfigError
+            raise ConfigError.new(file), "in #{file}: #{e.message}", e.backtrace
+        elsif !e.file
+            e.file = file
+            raise e, "in #{file}: #{e.message}", e.backtrace
+        else
+            raise e
+        end
+    end
 end
 
 

data/lib/autoproj/cmdline.rb

@@ -42,10 +42,14 @@ module Autoproj
                 Autoproj.prefix = Autoproj.user_config('prefix')
             end
 
+            if Autoproj.has_config_key?('randomize_layout')
+                @randomize_layout = Autoproj.user_config('randomize_layout')
+            end
+
             # If we are under rubygems, check that the GEM_HOME is right ...
             if $LOADED_FEATURES.any? { |l| l =~ /rubygems/ }
                 if ENV['GEM_HOME'] != Autoproj.gem_home
-                    raise ConfigError, "RubyGems is already loaded with a different GEM_HOME, make sure you are loading the right env.sh script !"
+                    raise ConfigError.new, "RubyGems is already loaded with a different GEM_HOME, make sure you are loading the right env.sh script !"
                 end
             end
 
@@ -71,14 +75,16 @@ module Autoproj
                 Autoproj.env_add 'PATH', bindir
             end
 
+            Autoproj.manifest = Manifest.new
+
             # We load the local init.rb first so that the manifest loading
             # process can use options defined there for the autoproj version
             # control information (for instance)
-            local_source = LocalSource.new
+            local_source = LocalPackageSet.new(Autoproj.manifest)
             Autoproj.load_if_present(local_source, local_source.local_dir, "init.rb")
 
             manifest_path = File.join(Autoproj.config_dir, 'manifest')
-            Autoproj.manifest = Manifest.load(manifest_path)
+            Autoproj.manifest.load(manifest_path)
 
             # Once thing left to do: handle the Autoproj.auto_update
             # configuration parameter. This has to be done here as the rest of
@@ -102,10 +108,10 @@ module Autoproj
             end
             # Do that AFTER we have properly setup Autoproj.osdeps as to avoid
             # unnecessarily redetecting the operating system
-	    Autoproj::OSDependencies.define_osdeps_mode_option
             if update_os_dependencies? || osdeps?
                 Autoproj.reset_option('operating_system')
             end
+	    Autoproj::OSDependencies.define_osdeps_mode_option
             Autoproj.osdeps.osdeps_mode
         end
 
@@ -137,6 +143,7 @@ module Autoproj
 
         def self.load_configuration(silent = false)
             manifest = Autoproj.manifest
+            manifest.cache_package_sets
 
             # Load init.rb files. each_source must not load the source.yml file, as
             # init.rb may define configuration options that are used there
@@ -177,7 +184,7 @@ module Autoproj
             if manifest.vcs
                 manifest.update_yourself
                 manifest_path = File.join(Autoproj.config_dir, 'manifest')
-                Autoproj.manifest = manifest = Manifest.load(manifest_path)
+                manifest.load(manifest_path)
             end
 
             source_os_dependencies = manifest.each_remote_source(false).
@@ -213,13 +220,20 @@ module Autoproj
             # First, we allow to user to specify packages based on disk paths, so
             # resolve those
             seen = Set.new
-            manifest.each_package_set do |name, packages, enabled_packages|
+            manifest.each_layout_level do |name, packages, enabled_packages|
                 packages -= seen
 
-                srcdir  = File.join(Autoproj.root_dir, name)
-                prefix  = File.join(Autoproj.build_dir, name)
-                logdir  = File.join(prefix, "log")
                 packages.each do |pkg_name|
+                    place =
+                        if randomize_layout?
+                            Digest::SHA256.hexdigest(pkg_name)[0, 12]
+                        else name
+                        end
+
+                    srcdir  = File.join(Autoproj.root_dir, place)
+                    prefix  = File.join(Autoproj.build_dir, place)
+                    logdir  = File.join(prefix, "log")
+
                     pkg = Autobuild::Package[pkg_name]
                     pkg.srcdir = File.join(srcdir, pkg_name)
                     pkg.prefix = prefix
@@ -237,6 +251,9 @@ module Autoproj
                 end
             end
 
+            # Resolve optional dependencies
+            manifest.resolve_optional_dependencies
+
             # Load the package's override files. each_source must not load the
             # source.yml file, as init.rb may define configuration options that are used
             # there
@@ -250,51 +267,97 @@ module Autoproj
         end
 
 
-        def self.display_sources(manifest)
+        def self.display_configuration(manifest, package_list = nil)
             # We can't have the Manifest class load the source.yml file, as it
             # cannot resolve all constants. So we need to do it ourselves to get
             # the name ...
-            sources = manifest.each_source(false).to_a
+            sets = manifest.each_package_set(false).to_a
 
-            if sources.empty?
+            if sets.empty?
                 Autoproj.progress("autoproj: no package sets defined in autoproj/manifest", :bold, :red)
+                return
+            end
+
+            all_packages = Hash.new
+            if package_list
+                package_sets = Set.new
+                package_list.each do |name|
+                    pkg_set = manifest.definition_source(name)
+                    package_sets << pkg_set
+                    all_packages[name] = [manifest.package(name).autobuild, pkg_set.name]
+                end
             else
-                Autoproj.progress("autoproj: available package sets", :bold)
-                manifest.each_source(false) do |source|
-                    source_yml = source.raw_description_file
-                    Autoproj.progress "  #{source_yml['name']}"
-                    if source.local?
-                        Autoproj.progress "    local source in #{source.local_dir}"
-                    else
-                        Autoproj.progress "    from:  #{source.vcs}"
-                        Autoproj.progress "    local: #{source.local_dir}"
+                package_sets = manifest.each_package_set
+                package_sets.each do |pkg_set|
+                    pkg_set.each_package.each do |pkg|
+                        all_packages[pkg.name] = [pkg, pkg_set.name]
                     end
+                end
+            end
 
-                    lines = []
-                    source.each_package.
-                        map { |pkg| [pkg.name, manifest.package_manifests[pkg.name]] }.
-                        sort_by { |name, _| name }.
-                        each do |name, source_manifest|
-                            vcs_def = manifest.importer_definition_for(name)
-                            if source_manifest
-                                lines << [name, source_manifest.short_documentation]
-                                lines << ["", vcs_def.to_s]
-                            else
-                                lines << [name, vcs_def.to_s]
-                            end
-                        end
+            Autoproj.progress
+            Autoproj.progress("autoproj: package sets", :bold)
+            package_sets.each do |pkg_set|
+                next if pkg_set.empty?
+                if pkg_set.imported_from
+                    Autoproj.progress "#{pkg_set.name} (imported by #{pkg_set.imported_from.name})"
+                else
+                    Autoproj.progress "#{pkg_set.name} (listed in manifest)"
+                end
+                if pkg_set.local?
+                    Autoproj.progress "  local set in #{pkg_set.local_dir}"
+                else
+                    Autoproj.progress "  from:  #{pkg_set.vcs}"
+                    Autoproj.progress "  local: #{pkg_set.local_dir}"
+                end
 
-                    w_col1, w_col2 = nil
-                    lines.each do |col1, col2|
-                        w_col1 = col1.size if !w_col1 || col1.size > w_col1
-                        w_col2 = col2.size if !w_col2 || col2.size > w_col2
+                imports = pkg_set.each_imported_set.to_a
+                if !imports.empty?
+                    Autoproj.progress "  imports #{imports.size} package sets"
+                    if !pkg_set.auto_imports?
+                        Autoproj.progress "    automatic imports are DISABLED for this set"
                     end
-                    puts "    packages:"
-                    format = "    | %-#{w_col1}s | %-#{w_col2}s |"
-                    lines.each do |col1, col2|
-                        puts(format % [col1, col2])
+                    imports.each do |imported_set|
+                        Autoproj.progress "    #{imported_set.name}"
                     end
                 end
+
+                set_packages = pkg_set.each_package.sort_by(&:name)
+                Autoproj.progress "  defines: #{set_packages.map(&:name).join(", ")}"
+            end
+
+            Autoproj.progress
+            Autoproj.progress("autoproj: packages", :bold)
+            all_packages.to_a.sort_by(&:first).map(&:last).each do |pkg, pkg_set|
+                if File.exists?(File.join(pkg.srcdir, "manifest.xml"))
+                    manifest.load_package_manifest(pkg.name)
+                    manifest.resolve_optional_dependencies
+                end
+
+                pkg_manifest = manifest.package_manifests[pkg.name];
+                vcs_def = manifest.importer_definition_for(pkg.name)
+                Autoproj.progress "#{pkg.name}#{": #{pkg_manifest.short_documentation}" if pkg_manifest && pkg_manifest.short_documentation}", :bold
+                Autoproj.progress "   defined in #{pkg_set}"
+                Autoproj.progress "   #{vcs_def.to_s}"
+
+                if !File.directory?(pkg.srcdir)
+                    Autoproj.progress "   NOT checked out yet, reported dependencies will be partial"
+                end
+
+                optdeps = pkg.optional_dependencies.to_set
+                real_deps = pkg.dependencies.to_a
+                actual_real_deps = real_deps.find_all { |dep_name| !optdeps.include?(dep_name) }
+                if !actual_real_deps.empty?
+                    Autoproj.progress "   deps: #{actual_real_deps.join(", ")}"
+                end
+
+                selected_opt_deps, opt_deps = optdeps.partition { |dep_name| real_deps.include?(dep_name) }
+                if !selected_opt_deps.empty?
+                    Autoproj.progress "   enabled opt deps: #{selected_opt_deps.join(", ")}"
+                end
+                if !opt_deps.empty?
+                    Autoproj.progress "   disabled opt deps: #{opt_deps.join(", ")}"
+                end
             end
         end
 
@@ -325,11 +388,11 @@ module Autoproj
 
         def self.verify_package_availability(pkg_name)
             if reason = Autoproj.manifest.exclusion_reason(pkg_name)
-                raise ConfigError, "#{pkg_name} is excluded from the build: #{reason}"
+                raise ConfigError.new, "#{pkg_name} is excluded from the build: #{reason}"
             end
             pkg = Autobuild::Package[pkg_name]
             if !pkg
-                raise ConfigError, "#{pkg_name} does not seem to exist"
+                raise ConfigError.new, "#{pkg_name} does not seem to exist"
             end
 
             # Verify that its dependencies are there, and add
@@ -349,7 +412,7 @@ module Autoproj
                 map do |pkg_name|
                     pkg = Autobuild::Package[pkg_name]
                     if !pkg
-                        raise ConfigError, "selected package #{pkg_name} does not exist"
+                        raise ConfigError.new, "selected package #{pkg_name} does not exist"
                     end
                     pkg
                 end.to_set
@@ -397,11 +460,13 @@ module Autoproj
                     # If the package has no importer, the source directory must
                     # be there
                     if !pkg.importer && !File.directory?(pkg.srcdir)
-                        raise ConfigError, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
+                        raise ConfigError.new, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
                     end
 
                     Rake::Task["#{pkg.name}-import"].invoke
                     manifest.load_package_manifest(pkg.name)
+                    pkg.resolve_optional_dependencies
+                    verify_package_availability(pkg.name)
                 end
 
                 current_packages.each do |pkg|
@@ -496,6 +561,7 @@ module Autoproj
         def self.check?; !!@check end
         def self.manifest_update?; !!@manifest_update end
         def self.only_config?; !!@only_config end
+        def self.randomize_layout?; !!@randomize_layout end
         def self.update_os_dependencies?
             # Check if the mode disables osdeps anyway ...
             if !@update_os_dependencies.nil? && !@update_os_dependencies
@@ -515,7 +581,7 @@ module Autoproj
         def self.partial_build?; !!@partial_build end
         def self.mail_config; @mail_config || Hash.new end
         def self.update_packages?; @mode == "update" || @mode == "envsh" || build? end
-        def self.update_envsh?; @mode == "envsh" || build? end
+        def self.update_envsh?; @mode == "envsh" || build? || @mode == "update" end
         def self.build?; @mode =~ /build/ end
         def self.doc?; @mode == "doc" end
         def self.snapshot?; @mode == "snapshot" end
@@ -531,7 +597,7 @@ module Autoproj
             end
         end
         def self.list_newest?; @list_newest end
-        def self.parse_arguments(args)
+        def self.parse_arguments(args, with_mode = true)
             @only_status = false
             @only_local  = false
             @show_osdeps = false
@@ -680,6 +746,10 @@ where 'mode' is one of:
                 opts.on("--local", "for status, do not access the network") do
                     @only_local = true
                 end
+                opts.on('--randomize-layout', 'in build and full-build, generate a random layout') do
+                    @randomize_layout = true
+                    Autoproj.change_option('randomize_layout', true)
+                end
 
                 opts.on("--verbose", "verbose output") do
                     Autoproj.verbose  = true
@@ -722,14 +792,16 @@ where 'mode' is one of:
             parser.parse!(args)
             @mail_config = mail_config
 
-            @mode = args.shift
-            unknown_mode = catch(:unknown) do
-                handle_mode(@mode, args)
-            end
-            if unknown_mode
-                STDERR.puts "unknown mode #{@mode}"
-                STDERR.puts "run autoproj --help for more documentation"
-                exit(1)
+            if with_mode
+                @mode = args.shift
+                unknown_mode = catch(:unknown) do
+                    handle_mode(@mode, args)
+                end
+                if unknown_mode
+                    STDERR.puts "unknown mode #{@mode}"
+                    STDERR.puts "run autoproj --help for more documentation"
+                    exit(1)
+                end
             end
 
             selection = args.dup
@@ -912,8 +984,8 @@ where 'mode' is one of:
             console = Autoproj.console
             
             sources = Autoproj.manifest.each_configuration_source.
-                map do |vcs, text_name, pkg_name, local_dir|
-                    Autoproj::Manifest.create_autobuild_package(vcs, text_name, pkg_name, local_dir)
+                map do |vcs, text_name, local_dir|
+                    Autoproj::Manifest.create_autobuild_package(vcs, text_name, local_dir)
                 end
 
             if !sources.empty?
@@ -1010,7 +1082,7 @@ where 'mode' is one of:
                     
                 FileUtils.mv config_dir, backup_name
             end
-            Autoproj::Manifest.update_source(vcs, "autoproj main configuration", 'autoproj_config', config_dir)
+            Autoproj::Manifest.update_package_set(vcs, "autoproj main configuration", config_dir)
 
             # Now write it in the config file
             File.open(File.join(Autoproj.config_dir, "config.yml"), "a") do |io|
@@ -1035,7 +1107,7 @@ manifest_source:
 
         def self.bootstrap(*args)
             if File.exists?(File.join("autoproj", "manifest"))
-                raise ConfigError, "this installation is already bootstrapped. Remove the autoproj directory if it is not the case"
+                raise ConfigError.new, "this installation is already bootstrapped. Remove the autoproj directory if it is not the case"
             end
 
             require 'set'
@@ -1096,7 +1168,7 @@ manifest_source:
                 manifest_data =
                     begin open(manifest_url) { |file| file.read }
                     rescue
-                        raise ConfigError, "cannot read #{manifest_url}, did you mean 'autoproj bootstrap VCSTYPE #{manifest_url}' ?"
+                        raise ConfigError.new, "cannot read #{manifest_url}, did you mean 'autoproj bootstrap VCSTYPE #{manifest_url}' ?"
                     end
 
                 File.open(File.join(Autoproj.config_dir, "manifest"), "w") do |io|
@@ -1109,6 +1181,8 @@ manifest_source:
                 do_switch_config(false, type, url, *options)
             end
 
+            Autoproj.save_config
+
             # Finally, generate an env.sh script
             File.open('env.sh', 'w') do |io|
                 io.write <<-EOSHELL