autoproj 1.5.7 → 1.5.8

data/History.txt

@@ -1,3 +1,13 @@
+= Version 1.5.8
+ * add a way to add some ignore regexp to the dependency tracking. Updated user
+   guide about that.
+ * small cosmetic fixes
+ * warn if an osdeps file overrides information present in other osdeps files
+ * performance improvement at initialization: we now cache the result of OS
+   autodetection in config.yml. That removes a few seconds due to calling
+   lsb_release
+ * better error reporting in some cases related to OS dependencies
+
 = Version 1.5.7
  * allow using options and constants in the manifest as well
    This is so that we can use variables in the VCS definition for package sets.

data/Manifest.txt

@@ -33,6 +33,7 @@ doc/guide/src/toc.page
 doc/guide/src/writing_manifest.page
 lib/autoproj.rb
 lib/autoproj/autobuild.rb
+lib/autoproj/base.rb
 lib/autoproj/cmdline.rb
 lib/autoproj/default.osdeps
 lib/autoproj/manifest.rb

data/bin/autoproj

@@ -10,20 +10,9 @@ require 'autoproj/autobuild'
 require 'open-uri'
 require 'autoproj/cmdline'
 
-require 'highline'
 include Autoproj
 
 InputError = Autoproj::InputError
-module Autoproj
-    @verbose = false
-    @console = HighLine.new
-
-    class << self
-        attr_accessor :verbose
-        attr_reader :console
-    end
-end
-
 selected_packages = Autoproj::CmdLine.parse_arguments(ARGV.dup)
 
 def color(*args)
@@ -39,6 +28,11 @@ def report(debug)
 rescue ConfigError => e
     STDERR.puts
     STDERR.puts color(e.message, :red, :bold)
+    root_dir = /^#{Regexp.quote(Autoproj.root_dir)}/
+    e.backtrace.find_all { |path| path =~ root_dir }.
+        each do |path|
+            STDERR.puts color("  in #{path}", :red, :bold)
+        end
     if debug then raise
     else exit 1
     end
@@ -122,8 +116,7 @@ report(Autobuild.debug) do
     # Install prepackaged dependencies needed to import and build the packages.
     # The required information is in the package sets configuration.
     if Autoproj::CmdLine.update_os_dependencies?
-        osdeps = manifest.known_os_packages
-        osdeps.install(Autoproj.build_system_dependencies)
+        Autoproj.osdeps.install(Autoproj.build_system_dependencies)
     end
 
     begin

data/doc/guide/src/autoproj_bootstrap

@@ -28,6 +28,14 @@ module Autoproj
     class << self
         attr_reader :verbose
     end
+
+    # Fake option management for the OSdeps class
+    def self.has_config_key?(*args)
+        false
+    end
+    def self.change_option(*args)
+        false
+    end
 end
 
 module Autobuild
@@ -69,14 +77,15 @@ require 'tempfile'
 module Autoproj
     class OSDependencies
         def self.load(file)
+            file = File.expand_path(file)
             begin
-                data = YAML.load(File.read(file))
+                data = YAML.load(File.read(file)) || Hash.new
                 verify_definitions(data)
             rescue ArgumentError => e
                 raise ConfigError, "error in #{file}: #{e.message}"
             end
 
-            OSDependencies.new(data)
+            OSDependencies.new(data, file)
         end
 
         class << self
@@ -98,33 +107,62 @@ module Autoproj
 
         AUTOPROJ_OSDEPS = File.join(File.expand_path(File.dirname(__FILE__)), 'default.osdeps')
         def self.load_default
-            if @default_osdeps
-                @default_osdeps
-            else
-                file = ENV['AUTOPROJ_DEFAULT_OSDEPS'] || AUTOPROJ_OSDEPS
-                if !File.file?(file)
-                    STDERR.puts "WARN: #{file} (from AUTOPROJ_DEFAULT_OSDEPS) is not a file, falling back to #{AUTOPROJ_OSDEPS}"
-                    file = AUTOPROJ_OSDEPS
-                end
-                @default_osdeps = OSDependencies.load(file)
+            file = ENV['AUTOPROJ_DEFAULT_OSDEPS'] || AUTOPROJ_OSDEPS
+            if !File.file?(file)
+                Autoproj.progress "WARN: #{file} (from AUTOPROJ_DEFAULT_OSDEPS) is not a file, falling back to #{AUTOPROJ_OSDEPS}"
+                file = AUTOPROJ_OSDEPS
             end
+            OSDependencies.load(file)
         end
 
+        # The information contained in the OSdeps files, as a hash
         attr_reader :definitions
+        # The information as to from which osdeps file the current package
+        # information in +definitions+ originates. It is a mapping from the
+        # package name to the osdeps file' full path
+        attr_reader :sources
+
+        # The Gem::SpecFetcher object that should be used to query RubyGems, and
+        # install RubyGems packages
         def gem_fetcher
             @gem_fetcher ||= Gem::SpecFetcher.fetcher
         end
 
-        def initialize(defs = Hash.new)
+        def initialize(defs = Hash.new, file = nil)
             @definitions = defs.to_hash
+            @sources     = Hash.new
+            if file
+                defs.each_key do |package_name|
+                    sources[package_name] = file
+                end
+            end
+        end
+
+        # Returns the full path to the osdeps file from which the package
+        # definition for +package_name+ has been taken
+        def source_of(package_name)
+            sources[package_name]
         end
 
+        # Merges the osdeps information of +info+ into +self+. If packages are
+        # defined in both OSDependencies objects, the information in +info+
+        # takes precedence
         def merge(info)
-            @definitions = definitions.merge(info.definitions)
+            root_dir = nil
+            @definitions = definitions.merge(info.definitions) do |h, v1, v2|
+                if v1 != v2
+                    root_dir ||= "#{Autoproj.root_dir}/"
+                    old = source_of(h).gsub(root_dir, '')
+                    new = info.source_of(h).gsub(root_dir, '')
+                    Autoproj.warn("osdeps definition for #{h}, previously defined in #{old} overriden by #{new}")
+                end
+                v2
+            end
+            @sources = sources.merge(info.sources)
         end
 
-        def self.verify_definitions(hash = nil)
-            hash ||= definitions
+        # Perform some sanity checks on the given osdeps definitions
+        def self.verify_definitions(hash)
             hash.each do |key, value|
                 if !key.kind_of?(String)
                     raise ArgumentError, "invalid osdeps definition: found an #{key.class}. Don't forget to put quotes around numbers"
@@ -153,6 +191,8 @@ module Autoproj
         def self.operating_system
             if @operating_system
                 return @operating_system
+            elsif Autoproj.has_config_key?('operating_system')
+                @operating_system = Autoproj.user_config('operating_system')
             elsif data = os_from_lsb
                 if data[0] != "debian"
                     # Fall back to reading debian_version, as
@@ -186,6 +226,8 @@ module Autoproj
             @operating_system =
                 [@operating_system[0].downcase,
                  @operating_system[1].map(&:downcase)]
+            Autoproj.change_option('operating_system', @operating_system, true)
+            @operating_system
         end
 
         def self.os_from_lsb
@@ -209,8 +251,8 @@ module Autoproj
         EOSCRIPT
 
         OS_PACKAGE_INSTALL = {
-            'debian' => 'apt-get install -y %s',
-            'ubuntu' => 'apt-get install -y %s',
+            'debian' => 'export DEBIAN_FRONTEND=noninteractive; apt-get install -y %s',
+            'ubuntu' => 'export DEBIAN_FRONTEND=noninteractive; apt-get install -y %s',
             'gentoo' => 'emerge --noreplace %s',
             'arch' => 'pacman -Sy --noconfirm %s'
         }
@@ -376,8 +418,8 @@ module Autoproj
             if !osdeps.empty?
                 shell_script = generate_os_script(osdeps)
                 if Autoproj.verbose
-                    STDERR.puts "Installing non-ruby OS dependencies with"
-                    STDERR.puts shell_script
+                    Autoproj.progress "Installing non-ruby OS dependencies with"
+                    Autoproj.progress shell_script
                 end
 
                 File.open('osdeps.sh', 'w') do |file|
@@ -393,6 +435,7 @@ module Autoproj
             end
 
             if !gems.empty?
+                Autobuild.progress "looking for RubyGems updates"
                 # Don't install gems that are already there ...
                 gems.delete_if do |name|
                     version_requirements = Gem::Requirement.default
@@ -416,8 +459,8 @@ module Autoproj
                 guess_gem_program
 
                 if Autoproj.verbose
-                    STDERR.puts "Installing rubygems dependencies with"
-                    STDERR.puts "gem install #{gems.join(" ")}"
+                    Autoproj.progress "Installing rubygems dependencies with"
+                    Autoproj.progress "gem install #{gems.join(" ")}"
                 end
                 Autobuild.progress "installing/updating RubyGems dependencies: #{gems.join(", ")}"
                 Autobuild::Subprocess.run 'autoproj', 'osdeps', Autobuild.tool('gem'), 'install', *gems
@@ -433,16 +476,50 @@ end
 
 DEFS = <<EODEFS
 --- 
+svn: 
+  arch: subversion
+  gentoo: dev-util/subversion
+  debian,ubuntu: subversion
+autobuild: gem
+zlib: 
+  debian,ubuntu: zlib1g-dev
+libxml2: 
+  arch: libxml2
+  gentoo: dev-libs/libxml2
+  debian,ubuntu: libxml2-dev
 none: ignore
+autotools: 
+  arch: automake autoconf
+  gentoo: 
+  - sys-devel/automake:1.9
+  - sys-devel/autoconf
+  debian,ubuntu: 
+  - automake1.9
+  - autoconf
+autoproj: gem
+archive: 
+  arch: 
+  - tar
+  - unzip
+  gentoo: 
+  - app-arch/tar
+  - app-arch/unzip
+  debian,ubuntu: 
+  - tar
+  - unzip
+lsb_release: 
+  arch: 
+  gentoo: sys-apps/lsb-release
+  debian,ubuntu: lsb-release
 ruby18: 
+  gentoo: 
+  - dev-lang/ruby:1.8
   debian,ubuntu: 
   - ruby1.8-dev
   - ruby1.8
   - rubygems1.8
   - ri1.8
   - libopenssl-ruby1.8
-  gentoo: 
-  - dev-lang/ruby:1.8
 ruby19: 
   debian: 
     squeeze,sid: 
@@ -453,65 +530,33 @@ ruby19:
     - ruby1.9.1
     - ruby1.9.1-dev
     - rubygems1.9.1
+  arch: 
+  - ruby
+  gentoo: 
+  - dev-lang/ruby:1.9
   ubuntu: 
   - ruby1.9.1
   - ruby1.9.1-dev
   - rubygems1.9.1
   - ri1.9.1
   - libopenssl-ruby1.9.1
-  gentoo: 
-  - dev-lang/ruby:1.9
-  arch: 
-  - ruby
-rdoc: gem
-build-essential: 
-  debian,ubuntu: build-essential
-  gentoo: 
-  arch: 
-libxml2: 
-  debian,ubuntu: libxml2-dev
-  gentoo: dev-libs/libxml2
-  arch: libxml2
-libxslt: 
-  debian,ubuntu: libxslt1-dev
-  gentoo: dev-libs/libxslt
-  arch: libxslt
-autobuild: gem
-autoproj: gem
 git: 
-  debian,ubuntu: git-core
-  gentoo: dev-vcs/git
   arch: git
-svn: 
-  debian,ubuntu: subversion
-  gentoo: dev-util/subversion
-  arch: subversion
+  gentoo: dev-vcs/git
+  debian,ubuntu: git-core
 cmake: 
-  debian,ubuntu: cmake
-  gentoo: dev-util/cmake
   arch: cmake
-autotools: 
-  debian,ubuntu: 
-  - automake1.9
-  - autoconf
-  gentoo: 
-  - sys-devel/automake:1.9
-  - sys-devel/autoconf
-  arch: automake autoconf
-lsb_release: 
-  debian,ubuntu: lsb-release
-  gentoo: sys-apps/lsb-release
+  gentoo: dev-util/cmake
+  debian,ubuntu: cmake
+build-essential: 
   arch: 
-archive: 
-  debian,ubuntu: 
-  - tar
-  - unzip
   gentoo: 
-  - app-arch/tar
-  - app-arch/unzip
-  arch: 
-  - tar
-  - unzip
+  debian,ubuntu: build-essential
+libxslt: 
+  arch: libxslt
+  gentoo: dev-libs/libxslt
+  debian,ubuntu: libxslt1-dev
+rdoc: gem
 
 EODEFS
 
@@ -531,7 +576,7 @@ rescue Autoproj::ConfigError => e
 end
 
 # Now try to find out the name of the gem binary
-PACKAGES      = %w{rdoc libxml2 libxslt build-essential lsb_release}
+PACKAGES      = %w{rdoc libxml2 libxslt zlib build-essential lsb_release}
 USER_PACKAGES = %w{autoproj}
 
 ENV['RUBYOPT']  = "-rubygems"

data/doc/guide/src/customization.page

@@ -127,6 +127,36 @@ overrides:
     branch: experimental
 {coderay}
 
+Tuning what files autoproj looks at to determine if a package should be updated
+-------------------------------------------------------------------------------
+When a package A depends on a package B, autoproj checks if some of the files in
+B's directory are newer than the latest build of A. If it is the case, it will
+trigger the build of B and then the one of A.
+
+Autoproj has a default list of files that it should ignore. Unfortunately, it
+may be that you are generating some files in the source directory that autoproj
+interprets as new files and trigger builds.
+
+If you have the impression that autoproj does too many rebuilds, run the build
+once with the <tt>--list-newest-files</tt> option. For instance,
+
+autoproj --list-newest-files fast-build
+{: .cmdline}
+
+If you find some files that should be ignored, add them either to the package
+sets (i.e. autoproj/remotes/blablab/init.rb) or in autoproj/init.rb with
+
+{coderay:: ruby}
+ignore /a_regular_expression/
+{coderay}
+
+where /a_regular_expression/ is a regular expression matching your files. For
+instance, to eliminate all files that have an extension in ".swp", one would do
+
+{coderay:: ruby}
+ignore /\.swp$/
+{coderay}
+
 Building local packages
 -----------------------
 

data/lib/autoproj.rb

@@ -1,9 +1,6 @@
-module Autoproj
-    class ConfigError < RuntimeError; end
-    class InternalError < RuntimeError; end
-end
-
 require "enumerator"
+require 'autobuild'
+require 'autoproj/base'
 require 'autoproj/version'
 require 'autoproj/manifest'
 require 'autoproj/osdeps'

data/lib/autoproj/autobuild.rb

@@ -23,7 +23,7 @@ module Autobuild
                         raise e
                     else
                         # Re-call osdeps to get a proper error message
-                        osdeps, gems = Autoproj.osdeps.partition_packages([name].to_set)
+                        osdeps, gems = Autoproj.osdeps.partition_packages([name].to_set, name => [self.name])
                         Autoproj.osdeps.resolve_os_dependencies(osdeps)
                     end
                 end
@@ -46,22 +46,19 @@ module Autoproj
     class Reporter < Autobuild::Reporter
         def error(error)
             error_lines = error.to_s.split("\n")
-            STDERR.puts color("Build failed: #{error_lines.shift}", :bold, :red)
-            STDERR.puts error_lines.join("\n")
+            Autoproj.progress("Build failed: #{error_lines.shift}", :bold, :red)
+            error_lines.each do |line|
+                Autoproj.progress line
+            end
         end
         def success
-            STDERR.puts color("Build finished successfully at #{Time.now}", :bold, :green)
+            Autoproj.progress("Build finished successfully at #{Time.now}", :bold, :green)
             if Autobuild.post_success_message
-                puts Autobuild.post_success_message
+                Autoproj.progress Autobuild.post_success_message
             end
         end
     end
 
-    # Displays a warning message
-    def self.warn(message)
-        STDERR.puts Autoproj.console.color("  WARN: #{message}", :magenta)
-    end
-
     @file_stack       = Array.new
 
     def self.package_name_from_options(spec)
@@ -111,6 +108,12 @@ module Autoproj
     end
 end
 
+def ignore(*paths)
+    paths.each do |p|
+        Autobuild.ignore(p)
+    end
+end
+
 # Common setup for packages
 def package_common(package_type, spec, &block) # :nodoc:
     package_name = Autoproj.package_name_from_options(spec)

data/lib/autoproj/base.rb

@@ -0,0 +1,6 @@
+module Autoproj
+    class ConfigError < RuntimeError; end
+    class InternalError < RuntimeError; end
+end
+
+

data/lib/autoproj/cmdline.rb

@@ -1,4 +1,30 @@
+require 'highline'
+require 'utilrb/module/attr_predicate'
 module Autoproj
+    class << self
+        attr_accessor :verbose
+        attr_reader :console
+        attr_predicate :silent?, true
+    end
+    @silent  = false
+    @verbose = false
+    @console = HighLine.new
+
+    def self.progress(*args)
+        if !silent?
+            if args.empty?
+                puts
+            else
+                STDERR.puts console.color(*args)
+            end
+        end
+    end
+
+    # Displays a warning message
+    def self.warn(message)
+        Autoproj.progress("  WARN: #{message}", :magenta)
+    end
+
     module CmdLine
         def self.initialize
             Autobuild::Reporting << Autoproj::Reporter.new
@@ -60,14 +86,17 @@ module Autoproj
             if Autoproj::CmdLine.update_os_dependencies.nil?
                 Autoproj::CmdLine.update_os_dependencies = manifest.auto_update?
             end
+
+            # Initialize the Autoproj.osdeps object by loading the default. The
+            # rest is loaded later
+            Autoproj.osdeps = Autoproj::OSDependencies.load_default
         end
 
         def self.update_myself
             return if !Autoproj::CmdLine.update_os_dependencies?
 
             # First things first, see if we need to update ourselves
-            osdeps = Autoproj::OSDependencies.load_default
-            if osdeps.install(%w{autobuild autoproj})
+            if Autoproj.osdeps.install(%w{autobuild autoproj})
                 # We updated autobuild or autoproj themselves ... Restart !
                 require 'rbconfig'
                 ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME']
@@ -75,7 +104,7 @@ module Autoproj
             end
         end
 
-        def self.load_configuration
+        def self.load_configuration(silent = false)
             manifest = Autoproj.manifest
 
             # Load init.rb files. each_source must not load the source.yml file, as
@@ -85,9 +114,11 @@ module Autoproj
             end
 
             # Load the required autobuild definitions
-            STDERR.puts color("autoproj: loading ...", :bold)
-            if !Autoproj.reconfigure?
-                STDERR.puts color("run 'autoproj --reconfigure' to change configuration values", :bold)
+            if !silent
+                Autoproj.progress("autoproj: loading ...", :bold)
+                if !Autoproj.reconfigure?
+                    Autoproj.progress("run 'autoproj --reconfigure' to change configuration values", :bold)
+                end
             end
             manifest.each_autobuild_file do |source, name|
                 Autoproj.import_autobuild_file source, name
@@ -109,7 +140,7 @@ module Autoproj
             Autoproj.save_config
 
             # Loads OS package definitions once and for all
-            Autoproj.osdeps = manifest.known_os_packages
+            Autoproj.load_osdeps_from_package_sets
         end
 
         def self.update_configuration
@@ -131,15 +162,14 @@ module Autoproj
 
             # Update the remote sources if there are any
             if manifest.has_remote_sources?
-                STDERR.puts color("autoproj: updating remote definitions of package sets", :bold)
+                Autoproj.progress("autoproj: updating remote definitions of package sets", :bold)
                 # If we need to install some packages to import our remote sources, do it
                 if update_os_dependencies?
-                    osdeps = OSDependencies.load_default
-                    osdeps.install(source_os_dependencies)
+                    Autoproj.osdeps.install(source_os_dependencies)
                 end
 
                 manifest.update_remote_sources
-                STDERR.puts
+                Autoproj.progress
             end
         end
 
@@ -186,17 +216,17 @@ module Autoproj
             sources = manifest.each_source(false).to_a
 
             if sources.empty?
-                STDERR.puts color("autoproj: no package sets defined in autoproj/manifest", :bold, :red)
+                Autoproj.progress("autoproj: no package sets defined in autoproj/manifest", :bold, :red)
             else
-                STDERR.puts color("autoproj: available package sets", :bold)
+                Autoproj.progress("autoproj: available package sets", :bold)
                 manifest.each_source(false) do |source|
                     source_yml = source.raw_description_file
-                    STDERR.puts "  #{source_yml['name']}"
+                    Autoproj.progress "  #{source_yml['name']}"
                     if source.local?
-                        puts "    local source in #{source.local_dir}"
+                        Autoproj.progress "    local source in #{source.local_dir}"
                     else
-                        puts "    from:  #{source.vcs}"
-                        puts "    local: #{source.local_dir}"
+                        Autoproj.progress "    from:  #{source.vcs}"
+                        Autoproj.progress "    local: #{source.local_dir}"
                     end
 
                     lines = []
@@ -236,18 +266,18 @@ module Autoproj
                 return manifest.default_packages
             end
             if selected_packages.empty? # no packages, terminate
-                STDERR.puts
-                STDERR.puts color("autoproj: no packages defined", :red)
+                Autoproj.progress
+                Autoproj.progress("autoproj: no packages defined", :red)
                 exit 0
             end
             selected_packages = selected_packages.to_set
 
             selected_packages = manifest.expand_package_selection(selected_packages)
             if selected_packages.empty?
-                STDERR.puts color("autoproj: wrong package selection on command line", :red)
+                Autoproj.progress("autoproj: wrong package selection on command line", :red)
                 exit 1
             elsif Autoproj.verbose
-                STDERR.puts "will install #{selected_packages.to_a.join(", ")}"
+                Autoproj.progress "will install #{selected_packages.to_a.join(", ")}"
             end
             selected_packages
         end
@@ -324,7 +354,27 @@ module Autoproj
             end
 
             if Autoproj.verbose
-                STDERR.puts "autoproj: finished importing packages"
+                Autoproj.progress "autoproj: finished importing packages"
+            end
+            if Autoproj::CmdLine.list_newest?
+                fields = []
+                Rake::Task.tasks.each do |task|
+                    if task.kind_of?(Autobuild::SourceTreeTask)
+                        task.timestamp
+                        fields << ["#{task.name}:", task.newest_file, task.newest_time.to_s]
+                    end
+                end
+
+                field_sizes = fields.inject([0, 0, 0]) do |sizes, line|
+                    3.times do |i|
+                        sizes[i] = [sizes[i], line[i].length].max
+                    end
+                    sizes
+                end
+                format = "  %-#{field_sizes[0]}s %-#{field_sizes[1]}s at %-#{field_sizes[2]}s"
+                fields.each do |line|
+                    Autoproj.progress(format % line)
+                end
             end
 
             return all_enabled_packages
@@ -332,9 +382,9 @@ module Autoproj
 
         def self.build_packages(selected_packages, all_enabled_packages)
             if Autoproj::CmdLine.doc?
-                STDERR.puts color("autoproj: building and installing documentation", :bold)
+                Autoproj.progress("autoproj: building and installing documentation", :bold)
             else
-                STDERR.puts color("autoproj: building and installing packages", :bold)
+                Autoproj.progress("autoproj: building and installing packages", :bold)
             end
 
             if Autoproj::CmdLine.update_os_dependencies?
@@ -367,15 +417,17 @@ module Autoproj
         class << self
             attr_accessor :update_os_dependencies
             attr_accessor :snapshot_dir
+            attr_writer :list_newest
         end
         def self.display_configuration?; !!@display_configuration end
         def self.force_re_build_with_depends?; !!@force_re_build_with_depends end
         def self.partial_build?; !!@partial_build end
-        def self.mail_config; @mail_config end
+        def self.mail_config; @mail_config || Hash.new end
         def self.update_packages?; @mode == "update" || @mode == "envsh" || build? end
         def self.build?; @mode =~ /build/ end
         def self.doc?; @mode == "doc" end
         def self.snapshot?; @mode == "snapshot" end
+        def self.list_newest?; @list_newest end
         def self.parse_arguments(args)
             @only_status = false
             @check = false
@@ -462,6 +514,9 @@ where 'mode' is one of:
                 opts.on("--with-depends", "apply rebuild and force-build to both packages selected on the command line and their dependencies") do
                     force_re_build_with_depends = true
                 end
+                opts.on("--list-newest", "for each source directory, list what is the newest file used by autoproj for dependency tracking") do
+                    Autoproj::CmdLine.list_newest = true
+                end
 
                 opts.on("--verbose", "verbose output") do
                     Autoproj.verbose  = true
@@ -663,21 +718,21 @@ where 'mode' is one of:
                 end
 
                 if last_was_in_sync
-                    STDERR.puts color(": local and remote are in sync", :green)
+                    Autoproj.progress(": local and remote are in sync", :green)
                 end
 
                 last_was_in_sync = false
                 STDERR.print "#{pkg_name}:"
 
                 if lines.size == 1
-                    STDERR.puts lines.first
+                    Autoproj.progress lines.first
                 else
-                    STDERR.puts
-                    STDERR.puts lines.join("\n")
+                    Autoproj.progress
+                    Autoproj.progress lines.join("\n")
                 end
             end
             if last_was_in_sync
-                STDERR.puts color(": local and remote are in sync", :green)
+                Autoproj.progress(": local and remote are in sync", :green)
             end
         end
 
@@ -690,13 +745,13 @@ where 'mode' is one of:
                 end
 
             if !sources.empty?
-                STDERR.puts color("autoproj: displaying status of configuration", :bold)
+                Autoproj.progress("autoproj: displaying status of configuration", :bold)
                 display_status(sources)
                 STDERR.puts
             end
 
 
-            STDERR.puts color("autoproj: displaying status of packages", :bold)
+            Autoproj.progress("autoproj: displaying status of packages", :bold)
             packages = packages.sort.map do |pkg_name|
                 Autobuild::Package[pkg_name]
             end
@@ -796,7 +851,7 @@ manifest_source:
             # assume that we are bootstrapping from another installation directory and
             # start by copying the .gems directory
             if ENV['GEM_HOME'] && ENV['GEM_HOME'] =~ /\.gems\/?$/ && ENV['GEM_HOME'] != File.join(Dir.pwd, ".gems")
-                STDERR.puts "autoproj: reusing bootstrap from #{File.dirname(ENV['GEM_HOME'])}"
+                Autoproj.progress "autoproj: reusing bootstrap from #{File.dirname(ENV['GEM_HOME'])}"
                 FileUtils.cp_r ENV['GEM_HOME'], ".gems"
                 ENV['GEM_HOME'] = File.join(Dir.pwd, ".gems")
 
@@ -814,7 +869,7 @@ manifest_source:
 
             if args.size == 1 # the user asks us to download a manifest
                 manifest_url = args.first
-                STDERR.puts color("autoproj: downloading manifest file #{manifest_url}", :bold)
+                Autoproj.progress("autoproj: downloading manifest file #{manifest_url}", :bold)
                 manifest_data =
                     begin open(manifest_url) { |file| file.read }
                     rescue
@@ -840,7 +895,7 @@ manifest_source:
                 EOSHELL
             end
 
-            STDERR.puts <<EOTEXT
+            Autoproj.progress <<EOTEXT
 
 add the following line at the bottom of your .bashrc:
   source #{Dir.pwd}/env.sh
@@ -889,8 +944,8 @@ EOTEXT
                 end
 
                 if !result.empty?
-                    STDERR.puts pkg.name
-                    STDERR.puts "  #{result.join("\n  ")}"
+                    Autoproj.progress pkg.name
+                    Autoproj.progress "  #{result.join("\n  ")}"
                 end
             end
         end
@@ -928,9 +983,9 @@ EOTEXT
                         io.write xml.to_xml
                     end
                     if !manifest
-                        STDERR.puts "created #{path}"
+                        Autoproj.progress "created #{path}"
                     else
-                        STDERR.puts "modified #{path}"
+                        Autoproj.progress "modified #{path}"
                     end
                 end
             end
@@ -949,10 +1004,10 @@ EOTEXT
                 package  = Autobuild::Package[package_name]
                 importer = package.importer
                 if !importer
-                    STDERR.puts "cannot snapshot #{package_name} as it has no importer"
+                    Autoproj.progress "cannot snapshot #{package_name} as it has no importer"
                     next
                 elsif !importer.respond_to?(:snapshot)
-                    STDERR.puts "cannot snapshot #{package_name} as the #{importer.class} importer does not support it"
+                    Autoproj.progress "cannot snapshot #{package_name} as the #{importer.class} importer does not support it"
                     next
                 end
 

data/lib/autoproj/default.osdeps

@@ -54,6 +54,10 @@ libxslt:
     gentoo: dev-libs/libxslt
     arch: libxslt
 
+zlib:
+    debian,ubuntu: zlib1g-dev
+
+
 autobuild: gem
 autoproj: gem
 

data/lib/autoproj/manifest.rb

@@ -202,7 +202,7 @@ module Autoproj
 
     def self.resolve_one_constant(name, value, result, definitions)
         result[name] = single_expansion(value, result) do |missing_name|
-            result[missing_name] = resolve_one_constant(missing_name, definitions.delete(missing_name), definitions)
+            result[missing_name] = resolve_one_constant(missing_name, definitions.delete(missing_name), result, definitions)
         end
     end
 
@@ -905,7 +905,10 @@ module Autoproj
             # file (we're not ready for that yet)
             sources = []
             each_remote_source(false) do |source|
-                Manifest.update_source(source.vcs, source.name || source.vcs.url, source.automatic_name, source.raw_local_dir)
+                name = if source.present? then source.name
+                       else source.vcs.url
+                       end
+                Manifest.update_source(source.vcs, name, source.automatic_name, source.raw_local_dir)
                 sources << source
             end
 
@@ -919,13 +922,35 @@ module Autoproj
             end
 
             remotes_symlinks_dir = File.join(Autoproj.config_dir, 'remotes')
-            FileUtils.rm_rf remotes_symlinks_dir
-            FileUtils.mkdir remotes_symlinks_dir
+            FileUtils.mkdir_p remotes_symlinks_dir
+            known_remotes = []
+
             # Create symbolic links from .remotes/weird_url to
             # autoproj/remotes/name. Explicitely load the source name first
             each_remote_source(false) do |source|
                 source.load_name
-                FileUtils.ln_sf source.raw_local_dir, File.join(remotes_symlinks_dir, source.name)
+                symlink_dest = File.join(remotes_symlinks_dir, source.name)
+
+                # Check if the current symlink is valid, and recreate it if it
+                # is not
+                if File.symlink?(symlink_dest)
+                    dest = File.readlink(symlink_dest)
+                    if dest != source.raw_local_dir
+                        FileUtils.rm_f symlink_dest
+                    end
+                else
+                    FileUtils.rm_f symlink_dest
+                    FileUtils.ln_sf source.raw_local_dir, symlink_dest
+                end
+
+                known_remotes << symlink_dest
+            end
+
+            # Now remove obsolete symlinks
+            Dir.glob(File.join(remotes_symlinks_dir, '*')).each do |file|
+                if File.symlink?(file) && !known_remotes.include?(file)
+                    FileUtils.rm_f file
+                end
             end
         end
 
@@ -1090,6 +1115,16 @@ module Autoproj
             names.to_set
         end
 
+        # Returns the package directory for the given package name
+        def whereis(package_name)
+            each_package_set do |layout_name, packages, _|
+                if packages.include?(package_name)
+                    return layout_name
+                end
+            end
+            raise ArgumentError, "cannot find #{package_name} in the current layout"
+        end
+
         # Loads the package's manifest.xml file for the current package
         #
         # Right now, the absence of a manifest makes autoproj only issue a
@@ -1130,17 +1165,6 @@ module Autoproj
             selected_packages.each(&:load_package_manifest)
         end
 
-        # Returns an OSDependencies instance that defined the known OS packages,
-        # as well as how to install them
-        def known_os_packages
-            osdeps = OSDependencies.load_default
-
-            each_osdeps_file do |source, file|
-                osdeps.merge(OSDependencies.load(file))
-            end
-            osdeps
-        end
-
         def install_os_dependencies(packages)
             required_os_packages = Set.new
             package_os_deps = Hash.new { |h, k| h[k] = Array.new }
@@ -1156,7 +1180,7 @@ module Autoproj
                 end
             end
 
-            known_os_packages.install(required_os_packages, package_os_deps)
+            Autoproj.osdeps.install(required_os_packages, package_os_deps)
         end
 
         # Package selection can be done in three ways:
@@ -1228,6 +1252,13 @@ module Autoproj
         attr_accessor :osdeps
     end
 
+    def self.load_osdeps_from_package_sets
+        manifest.each_osdeps_file do |source, file|
+            osdeps.merge(OSDependencies.load(file))
+        end
+        osdeps
+    end
+
     class PackageManifest
         def self.load(package, file)
             doc = Nokogiri::XML(File.read(file)) do |c|

data/lib/autoproj/options.rb

@@ -24,7 +24,8 @@ module Autoproj
 
         def ask(current_value)
             default_value = if current_value then current_value.to_s
-                            else options[:default].to_str
+                            elsif options[:default] then options[:default].to_str
+                            else ''
                             end
 
             STDERR.print "  #{doc} [#{default_value}] "
@@ -35,7 +36,7 @@ module Autoproj
             validate(answer)
 
         rescue InputError => e
-            STDERR.puts Autoproj.console.color("invalid value: #{e.message}", :red)
+            Autoproj.progress("invalid value: #{e.message}", :red)
             retry
         end
 
@@ -89,7 +90,7 @@ module Autoproj
             value = configure(key)
         else
             if !seen
-                STDERR.puts "  #{@declared_options[key].doc}: #{value}"
+                Autoproj.progress "  #{@declared_options[key].doc}: #{value}"
                 @user_config[key] = [value, true]
             end
             value

data/lib/autoproj/osdeps.rb

@@ -2,14 +2,15 @@ require 'tempfile'
 module Autoproj
     class OSDependencies
         def self.load(file)
+            file = File.expand_path(file)
             begin
-                data = YAML.load(File.read(file))
+                data = YAML.load(File.read(file)) || Hash.new
                 verify_definitions(data)
             rescue ArgumentError => e
                 raise ConfigError, "error in #{file}: #{e.message}"
             end
 
-            OSDependencies.new(data)
+            OSDependencies.new(data, file)
         end
 
         class << self
@@ -31,33 +32,62 @@ module Autoproj
 
         AUTOPROJ_OSDEPS = File.join(File.expand_path(File.dirname(__FILE__)), 'default.osdeps')
         def self.load_default
-            if @default_osdeps
-                @default_osdeps
-            else
-                file = ENV['AUTOPROJ_DEFAULT_OSDEPS'] || AUTOPROJ_OSDEPS
-                if !File.file?(file)
-                    STDERR.puts "WARN: #{file} (from AUTOPROJ_DEFAULT_OSDEPS) is not a file, falling back to #{AUTOPROJ_OSDEPS}"
-                    file = AUTOPROJ_OSDEPS
-                end
-                @default_osdeps = OSDependencies.load(file)
+            file = ENV['AUTOPROJ_DEFAULT_OSDEPS'] || AUTOPROJ_OSDEPS
+            if !File.file?(file)
+                Autoproj.progress "WARN: #{file} (from AUTOPROJ_DEFAULT_OSDEPS) is not a file, falling back to #{AUTOPROJ_OSDEPS}"
+                file = AUTOPROJ_OSDEPS
             end
+            OSDependencies.load(file)
         end
 
+        # The information contained in the OSdeps files, as a hash
         attr_reader :definitions
+        # The information as to from which osdeps file the current package
+        # information in +definitions+ originates. It is a mapping from the
+        # package name to the osdeps file' full path
+        attr_reader :sources
+
+        # The Gem::SpecFetcher object that should be used to query RubyGems, and
+        # install RubyGems packages
         def gem_fetcher
             @gem_fetcher ||= Gem::SpecFetcher.fetcher
         end
 
-        def initialize(defs = Hash.new)
+        def initialize(defs = Hash.new, file = nil)
             @definitions = defs.to_hash
+            @sources     = Hash.new
+            if file
+                defs.each_key do |package_name|
+                    sources[package_name] = file
+                end
+            end
         end
 
+        # Returns the full path to the osdeps file from which the package
+        # definition for +package_name+ has been taken
+        def source_of(package_name)
+            sources[package_name]
+        end
+
+        # Merges the osdeps information of +info+ into +self+. If packages are
+        # defined in both OSDependencies objects, the information in +info+
+        # takes precedence
         def merge(info)
-            @definitions = definitions.merge(info.definitions)
+            root_dir = nil
+            @definitions = definitions.merge(info.definitions) do |h, v1, v2|
+                if v1 != v2
+                    root_dir ||= "#{Autoproj.root_dir}/"
+                    old = source_of(h).gsub(root_dir, '')
+                    new = info.source_of(h).gsub(root_dir, '')
+                    Autoproj.warn("osdeps definition for #{h}, previously defined in #{old} overriden by #{new}")
+                end
+                v2
+            end
+            @sources = sources.merge(info.sources)
         end
 
-        def self.verify_definitions(hash = nil)
-            hash ||= definitions
+        # Perform some sanity checks on the given osdeps definitions
+        def self.verify_definitions(hash)
             hash.each do |key, value|
                 if !key.kind_of?(String)
                     raise ArgumentError, "invalid osdeps definition: found an #{key.class}. Don't forget to put quotes around numbers"
@@ -86,6 +116,8 @@ module Autoproj
         def self.operating_system
             if @operating_system
                 return @operating_system
+            elsif Autoproj.has_config_key?('operating_system')
+                @operating_system = Autoproj.user_config('operating_system')
             elsif data = os_from_lsb
                 if data[0] != "debian"
                     # Fall back to reading debian_version, as
@@ -119,6 +151,8 @@ module Autoproj
             @operating_system =
                 [@operating_system[0].downcase,
                  @operating_system[1].map(&:downcase)]
+            Autoproj.change_option('operating_system', @operating_system, true)
+            @operating_system
         end
 
         def self.os_from_lsb
@@ -142,8 +176,8 @@ module Autoproj
         EOSCRIPT
 
         OS_PACKAGE_INSTALL = {
-            'debian' => 'apt-get install -y %s',
-            'ubuntu' => 'apt-get install -y %s',
+            'debian' => 'export DEBIAN_FRONTEND=noninteractive; apt-get install -y %s',
+            'ubuntu' => 'export DEBIAN_FRONTEND=noninteractive; apt-get install -y %s',
             'gentoo' => 'emerge --noreplace %s',
             'arch' => 'pacman -Sy --noconfirm %s'
         }
@@ -309,8 +343,8 @@ module Autoproj
             if !osdeps.empty?
                 shell_script = generate_os_script(osdeps)
                 if Autoproj.verbose
-                    STDERR.puts "Installing non-ruby OS dependencies with"
-                    STDERR.puts shell_script
+                    Autoproj.progress "Installing non-ruby OS dependencies with"
+                    Autoproj.progress shell_script
                 end
 
                 File.open('osdeps.sh', 'w') do |file|
@@ -350,8 +384,8 @@ module Autoproj
                 guess_gem_program
 
                 if Autoproj.verbose
-                    STDERR.puts "Installing rubygems dependencies with"
-                    STDERR.puts "gem install #{gems.join(" ")}"
+                    Autoproj.progress "Installing rubygems dependencies with"
+                    Autoproj.progress "gem install #{gems.join(" ")}"
                 end
                 Autobuild.progress "installing/updating RubyGems dependencies: #{gems.join(", ")}"
                 Autobuild::Subprocess.run 'autoproj', 'osdeps', Autobuild.tool('gem'), 'install', *gems

data/lib/autoproj/system.rb

@@ -141,7 +141,7 @@ module Autoproj
 
             output = `ldd -r #{name} 2>&1`
             if output =~ /undefined symbol/
-                STDERR.puts Autoproj.console.color("WARN: #{name} has undefined symbols", :magenta)
+                Autoproj.progress("WARN: #{name} has undefined symbols", :magenta)
             end
         end
     end

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "1.5.7"
+    VERSION = "1.5.8"
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 1
   - 5
-  - 7
-  version: 1.5.7
+  - 8
+  version: 1.5.8
 platform: ruby
 authors: 
 - Sylvain Joyeux
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-08 00:00:00 +02:00
+date: 2010-07-15 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -214,6 +214,7 @@ files:
 - doc/guide/src/writing_manifest.page
 - lib/autoproj.rb
 - lib/autoproj/autobuild.rb
+- lib/autoproj/base.rb
 - lib/autoproj/cmdline.rb
 - lib/autoproj/default.osdeps
 - lib/autoproj/manifest.rb