autoproj 1.6.1 → 1.6.2.rc2

data/lib/autoproj/cmdline.rb

@@ -30,54 +30,6 @@ module Autoproj
     end
 
     module CmdLine
-	def self.handle_automatic_osdeps
-            if !Autoproj.has_config_key?('automatic_osdeps') && ENV['AUTOPROJ_AUTOMATIC_OSDEPS']
-                mode = ENV['AUTOPROJ_AUTOMATIC_OSDEPS']
-                mode =
-                    if mode == 'true'     then Autoproj::OSDependencies::AUTOMATIC
-                    elsif mode == 'false' then Autoproj::OSDependencies::MANUAL
-                    elsif mode == 'wait'  then Autoproj::OSDependencies::WAIT
-                    else Autoproj::OSDependencies::ASK
-                    end
-
-                Autoproj.change_option('automatic_osdeps', mode, true)
-            end
-
-            short_doc = "Should autoproj handle the OS package installation automatically (yes, no, wait or ask) ?"
-            long_doc =<<-EOT
-* if you say "yes", the OS dependencies will be handled by autoproj.
-* if you say "no", the list of OS dependencies that need to be installed will be
-  listed, and autoproj will go on, assuming that you have installed them yourself.
-* if you say "ask", you will be prompted each time a package needs to be installed.
-* if you say "wait", autoproj will simply wait for you to press ENTER to continue
-  after it prompted you for the dependencies.
-    
-This value can be changed anytime by calling an autoproj operation
-with the --reconfigure option (e.g. autoproj update --reconfigure).
-Moreover, the "autoproj osdeps" call will always allow you to install
-OS dependencies through autoproj.
-            EOT
-            long_doc = long_doc.strip
-
-	    Autoproj.configuration_option 'automatic_osdeps', 'string',
-		:default => 'yes',
-		:doc => [short_doc, long_doc] do |value|
-                    begin
-                        Autoproj::BuildOption.validate_boolean(value, Hash.new)
-                    rescue Autoproj::InputError
-                        if value.to_s == "ask"
-                            :ask
-                        elsif value.to_s == "wait"
-                            :wait
-                        else
-                            raise Autoproj::InputError, "invalid value. Please answer 'yes', 'no', 'wait' or 'ask' -- without the quotes"
-                        end
-                    end
-                end
-
-	    Autoproj.user_config('automatic_osdeps')
-	end
-
         def self.initialize
             Autobuild::Reporting << Autoproj::Reporter.new
             if mail_config[:to]
@@ -106,8 +58,6 @@ OS dependencies through autoproj.
             Autobuild.srcdir  = Autoproj.root_dir
             Autobuild.logdir = File.join(Autobuild.prefix, 'log')
 
-	    handle_automatic_osdeps
-
             ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME']
             if ruby != 'ruby'
                 bindir = File.join(Autoproj.build_dir, 'bin')
@@ -141,24 +91,44 @@ OS dependencies through autoproj.
             if Autobuild.do_update.nil?
                 Autobuild.do_update = manifest.auto_update?
             end
-            if Autoproj::CmdLine.update_os_dependencies.nil?
-                Autoproj::CmdLine.update_os_dependencies = manifest.auto_update?
-            end
 
             # Initialize the Autoproj.osdeps object by loading the default. The
             # rest is loaded later
             Autoproj.osdeps = Autoproj::OSDependencies.load_default
+            Autoproj.osdeps.silent = !osdeps?
+            Autoproj.osdeps.filter_uptodate_packages = osdeps_filter_uptodate?
+            if Autoproj::CmdLine.osdeps_forced_mode
+                Autoproj.osdeps.osdeps_mode = Autoproj::CmdLine.osdeps_forced_mode
+            end
+            if @update_os_dependencies || Autoproj::CmdLine.osdeps?
+                Autoproj.reset_option('operating_system')
+            end
+            # Do that AFTER we have properly setup Autoproj.osdeps as to avoid
+            # unnecessarily redetecting the operating system
+	    Autoproj::OSDependencies.define_osdeps_mode_option
+            Autoproj.osdeps.osdeps_mode
         end
 
         def self.update_myself
             return if !Autoproj::CmdLine.update_os_dependencies?
 
+            # This is a guard to avoid infinite recursion in case the user is
+            # running autoproj osdeps --force
+            if ENV['AUTOPROJ_RESTARTING'] == '1'
+                return
+            end
+
             # First things first, see if we need to update ourselves
             if Autoproj.osdeps.install(%w{autobuild autoproj})
+                puts
+                Autoproj.progress 'autoproj and/or autobuild has been updated, restarting autoproj'
+                puts
+
                 # We updated autobuild or autoproj themselves ... Restart !
                 #
                 # ...But first save the configuration (!)
                 Autoproj.save_config
+                ENV['AUTOPROJ_RESTARTING'] = '1'
                 require 'rbconfig'
                 ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME']
                 exec(ruby, $0, *ARGV)
@@ -217,13 +187,18 @@ OS dependencies through autoproj.
 
             # Update the remote sources if there are any
             if manifest.has_remote_sources?
-                Autoproj.progress("autoproj: updating remote definitions of package sets", :bold)
+                if manifest.should_update_remote_sources
+                    Autoproj.progress("autoproj: updating remote definitions of package sets", :bold)
+                end
+
                 # If we need to install some packages to import our remote sources, do it
                 if update_os_dependencies?
                     Autoproj.osdeps.install(source_os_dependencies)
                 end
 
-                manifest.update_remote_sources
+                if manifest.should_update_remote_sources
+                    manifest.update_remote_sources
+                end
                 Autoproj.progress
             end
         end
@@ -389,7 +364,9 @@ OS dependencies through autoproj.
                     delete_if { |pkg_name, pkg| Autoproj.manifest.excluded?(pkg_name) || Autoproj.manifest.ignored?(pkg_name) }
 
                 packages.each do |_, pkg|
-                    pkg.import
+                    pkg.isolate_errors do
+                        pkg.import
+                    end
                 end
 
             ensure
@@ -498,10 +475,19 @@ OS dependencies through autoproj.
         def self.manifest; Autoproj.manifest end
         def self.bootstrap?; !!@bootstrap end
         def self.only_status?; !!@only_status end
+        def self.only_local?; !!@only_local end
         def self.check?; !!@check end
         def self.manifest_update?; !!@manifest_update end
         def self.only_config?; !!@only_config end
-        def self.update_os_dependencies?; !!@update_os_dependencies end
+        def self.update_os_dependencies?
+            # Check if the mode disables osdeps anyway ...
+            if !@update_os_dependencies.nil? && !@update_os_dependencies
+                return false
+            end
+
+            # Now look for what the user wants
+            Autoproj.osdeps.osdeps_mode != 'none' || !Autoproj.osdeps.silent?
+        end
         class << self
             attr_accessor :update_os_dependencies
             attr_accessor :snapshot_dir
@@ -515,14 +501,29 @@ OS dependencies through autoproj.
         def self.build?; @mode =~ /build/ end
         def self.doc?; @mode == "doc" end
         def self.snapshot?; @mode == "snapshot" end
+
+        def self.osdeps?; @mode == "osdeps" end
+        def self.show_osdeps?; @mode == "osdeps" && @show_osdeps end
+        def self.revshow_osdeps?; @mode == "osdeps" && @revshow_osdeps end
+        def self.osdeps_forced_mode; @osdeps_forced_mode end
+        def self.osdeps_filter_uptodate?
+            if @mode == "osdeps"
+                @osdeps_filter_uptodate
+            else true
+            end
+        end
         def self.list_newest?; @list_newest end
         def self.parse_arguments(args)
             @only_status = false
+            @only_local  = false
+            @show_osdeps = false
+            @revshow_osdeps = false
+            @osdeps_filter_uptodate = true
+            @osdeps_forced_mode = nil
             @check = false
             @manifest_update = false
             @display_configuration = false
-            @update_os_dependencies = true
-            update_os_dependencies  = nil
+            @update_os_dependencies = nil
             @force_re_build_with_depends = false
             force_re_build_with_depends = nil
             @only_config = false
@@ -586,7 +587,10 @@ where 'mode' is one of:
                 opts.on("--[no-]update", "[do not] update already checked-out packages (build modes only)") do |value|
                     do_update = value
                 end
-                opts.on("--os", "displays the operating system as detected by autoproj") do
+                opts.on("--keep-going", "-k", "continue building even though one package has an error") do
+                    Autobuild.ignore_errors = true
+                end
+                opts.on("--os-version", "displays the operating system as detected by autoproj") do
                     os = OSDependencies.operating_system
                     if !os
                         puts "no information about that OS"
@@ -600,15 +604,49 @@ where 'mode' is one of:
                     exit 0
                 end
 
-                opts.on("--[no-]osdeps", "[do not] install prepackaged dependencies (build and update modes only)") do |value|
-                    update_os_dependencies = value
-                end
                 opts.on("--with-depends", "apply rebuild and force-build to both packages selected on the command line and their dependencies") do
                     force_re_build_with_depends = true
                 end
                 opts.on("--list-newest", "for each source directory, list what is the newest file used by autoproj for dependency tracking") do
                     Autoproj::CmdLine.list_newest = true
                 end
+                opts.on("--rshow", "in the osdeps mode, shows information for each OS package") do
+                    @revshow_osdeps = true
+                end
+                opts.on("--show", "in the osdeps mode, show a per-package listing of the OS dependencies instead of installing them") do
+                    @show_osdeps = true
+                end
+                opts.on("--no-osdeps", "disable osdeps handling in build and update modes") do |value|
+                    @osdeps_forced_mode = 'none'
+                end
+                opts.on("--all", "in osdeps mode, install both OS packages and RubyGem packages, regardless of the otherwise selected mode") do
+                    @osdeps_forced_mode = 'all'
+                end
+                opts.on("--os", "in osdeps mode, install OS packages and display information about the RubyGem packages, regardless of the otherwise selected mode") do
+                    if @osdeps_forced_mode == 'ruby'
+                        # Make --ruby --os behave like --all
+                        @osdeps_forced_mode = 'all'
+                    else
+                        @osdeps_forced_mode = 'os'
+                    end
+                end
+                opts.on('--force', 'in osdeps mode, do not filter out installed and uptodate packages') do
+                    @osdeps_filter_uptodate = false
+                end
+                opts.on("--ruby", "in osdeps mode, install only RubyGem packages and display information about the OS packages, regardless of the otherwise selected mode") do
+                    if @osdeps_forced_mode == 'os'
+                        # Make --ruby --os behave like --all
+                        @osdeps_forced_mode = 'all'
+                    else
+                        @osdeps_forced_mode = 'ruby'
+                    end
+                end
+                opts.on("--none", "in osdeps mode, do not install any package but display information about them, regardless of the otherwise selected mode") do
+                    @osdeps_forced_mode = 'none'
+                end
+                opts.on("--local", "for status, do not access the network") do
+                    @only_local = true
+                end
 
                 opts.on("--verbose", "verbose output") do
                     Autoproj.verbose  = true
@@ -663,7 +701,6 @@ where 'mode' is one of:
 
             selection = args.dup
             @partial_build = !selection.empty?
-            @update_os_dependencies = update_os_dependencies if !update_os_dependencies.nil?
             @force_re_build_with_depends = force_re_build_with_depends if !force_re_build_with_depends.nil?
             Autobuild.do_update = do_update if !do_update.nil?
             selection
@@ -717,7 +754,6 @@ where 'mode' is one of:
                 @update_os_dependencies = true
             when "update"
                 Autobuild.do_update = true
-                @update_os_dependencies = true
                 Autobuild.do_build  = false
             when "check"
                 Autobuild.do_update = false
@@ -733,7 +769,6 @@ where 'mode' is one of:
                 Autobuild.do_update = false
                 @update_os_dependencies = true
                 Autobuild.do_build  = false
-                Autoproj::OSDependencies.force_osdeps = true
             when "status"
                 @only_status = true
                 Autobuild.do_update = false
@@ -745,7 +780,7 @@ where 'mode' is one of:
             when "update-config"
                 @only_config = true
                 Autobuild.do_update = true
-                @update_os_dependencies = false
+                @update_os_dependencies = true
                 Autobuild.do_build = false
             when "list-config"
                 @only_config = true
@@ -780,7 +815,7 @@ where 'mode' is one of:
                 elsif !File.directory?(pkg.srcdir)
                     lines << Autoproj.color("  is not imported yet", :magenta)
                 else
-                    status = pkg.importer.status(pkg)
+                    status = pkg.importer.status(pkg,@only_local)
                     if status.uncommitted_code
                         lines << Autoproj.color("  contains uncommitted modifications", :red)
                     end
@@ -924,8 +959,9 @@ where 'mode' is one of:
             vcs = Autoproj.normalize_vcs_definition(vcs_def)
 
             # Install the OS dependencies required for this VCS
-            handle_automatic_osdeps
+	    Autoproj::OSDependencies.define_osdeps_mode_option
             osdeps = Autoproj::OSDependencies.load_default
+            osdeps.osdeps_mode
             osdeps.install([vcs.type])
 
             # Now check out the actual configuration
@@ -968,7 +1004,29 @@ manifest_source:
             if File.exists?(File.join("autoproj", "manifest"))
                 raise ConfigError, "this installation is already bootstrapped. Remove the autoproj directory if it is not the case"
             end
-            Autobuild.logdir = File.join('build', 'log')
+
+            require 'set'
+            curdir_entries = Dir.entries('.').to_set - [".", "..", "autoproj_bootstrap", ".gems", 'env.sh'].to_set
+            if !curdir_entries.empty? && ENV['AUTOPROJ_BOOTSTRAP_IGNORE_NONEMPTY_DIR'] != '1'
+                while true
+                    print "The current directory is not empty, continue bootstrapping anyway ? [yes] "
+                    STDOUT.flush
+                    answer = STDIN.readline.chomp
+                    if answer == "no"
+                        exit
+                    elsif answer == "" || answer == "yes"
+                        # Set this environment variable since we might restart
+                        # autoproj later on.
+                        ENV['AUTOPROJ_BOOTSTRAP_IGNORE_NONEMPTY_DIR'] = '1'
+                        break
+                    else
+                        STDOUT.puts "invalid answer. Please answer 'yes' or 'no'"
+                        STDOUT.flush
+                    end
+                end
+            end
+
+            Autobuild.logdir = File.join(Autoproj.prefix, 'log')
 
             # Check if GEM_HOME is set. If it is the case, assume that we are
             # bootstrapping from another installation directory and start by
@@ -1155,6 +1213,111 @@ export PATH=$GEM_HOME/bin:$PATH
                 io.write YAML.dump(overrides)
             end
         end
+
+        # Displays the reverse OS dependencies (i.e. for each osdeps package,
+        # who depends on it and where it is defined)
+        def self.revshow_osdeps(packages)
+            _, ospkg_to_pkg = Autoproj.manifest.list_os_dependencies(packages)
+
+            # A mapping from a package name to
+            #   [is_os_pkg, is_gem_pkg, definitions, used_by]
+            #
+            # where 
+            #
+            # +used_by+ is the set of autobuild package names that use this
+            # osdeps package
+            #
+            # +definitions+ is a osdep_name => definition_file mapping
+            mapping = Hash.new { |h, k| h[k] = [false, false, Hash.new, Set.new] }
+
+            ospkg_to_pkg.each do |pkg_osdep, pkgs|
+                osdeps, gems = Autoproj.osdeps.
+                    partition_packages([pkg_osdep], ospkg_to_pkg)
+
+                gems.each do |gem_name|
+                    mapping[gem_name][1] = true
+                    mapping[gem_name][2][pkg_osdep] = Autoproj.osdeps.source_of(pkg_osdep)
+                    mapping[gem_name][3] |= pkgs
+                end
+
+                if Autoproj::OSDependencies.supported_operating_system?
+                    osdeps = Autoproj.osdeps.
+                        resolve_os_dependencies(osdeps)
+                end
+                osdeps.each do |ospkg_name|
+                    mapping[ospkg_name][0] = true
+                    mapping[ospkg_name][2][pkg_osdep] = Autoproj.osdeps.source_of(pkg_osdep)
+                    mapping[ospkg_name][3] |= pkgs
+                end
+            end
+
+            mapping = mapping.sort_by(&:first)
+            mapping.each do |pkg_name, (is_os_pkg, is_gem_pkg, definitions, used_by)|
+                kind = if is_os_pkg && is_gem_pkg
+                           "both a RubyGem and OS package"
+                       elsif is_os_pkg
+                           "an OS package"
+                       else
+                           "a RubyGem package"
+                       end
+
+                puts "#{pkg_name} is #{kind}"
+                definitions.to_a.
+                    sort_by(&:first).
+                    each do |osdep_name, file_name|
+                        puts "  defined as #{osdep_name} in #{file_name}"
+                    end
+
+                puts "  depended-upon by #{used_by.sort.join(", ")}"
+            end
+        end
+
+        # Displays the OS dependencies required by the given packages
+        def self.show_osdeps(packages)
+            _, ospkg_to_pkg = Autoproj.manifest.list_os_dependencies(packages)
+
+            # ospkg_to_pkg is the reverse mapping to what we want. Invert it
+            mapping = Hash.new { |h, k| h[k] = Set.new }
+            ospkg_to_pkg.each do |ospkg, pkgs|
+                pkgs.each do |pkg_name|
+                    mapping[pkg_name] << ospkg
+                end
+            end
+        
+            # Now sort it by package name (better for display)
+            package_osdeps = mapping.to_a.
+                sort_by { |name, _| name }
+
+            package_osdeps.each do |pkg_name, pkg_osdeps|
+                if pkg_osdeps.empty?
+                    puts "  #{pkg_name}: no OS dependencies"
+                    next
+                end
+
+                osdeps, gems = Autoproj.osdeps.
+                    partition_packages(pkg_osdeps, ospkg_to_pkg)
+
+                puts "  #{pkg_name}:"
+                if !gems.empty?
+                    puts "    RubyGem packages: #{gems.to_a.sort.join(", ")}"
+                end
+
+                # If we are on a supported OS, convert the osdeps name to plain
+                # package name
+                if Autoproj::OSDependencies.supported_operating_system?
+                    pkg_osdeps = Autoproj.osdeps.
+                        resolve_os_dependencies(osdeps)
+
+                    if !pkg_osdeps.empty?
+                        puts "    OS packages:      #{pkg_osdeps.to_a.sort.join(", ")}"
+                    end
+                else
+                    if !os_packages.empty?
+                        puts "    OS dependencies:  #{os_packages.to_a.sort.join(", ")}"
+                    end
+                end
+            end
+        end
     end
 end
 

data/lib/autoproj/manifest.rb

@@ -616,6 +616,15 @@ module Autoproj
         # The path to the manifest file that has been loaded
         attr_reader :file
 
+        # True if osdeps should be handled in update and build, or left to the
+        # osdeps command
+        def auto_osdeps?
+            if data.has_key?('auto_osdeps')
+                !!data['auto_osdeps']
+            else true
+            end
+        end
+
         # True if autoproj should run an update automatically when the user
         # uses" build"
         def auto_update?
@@ -721,7 +730,7 @@ module Autoproj
             end
 
             if source_name && !done_something
-                raise ConfigError, "source '#{source_name}' does not exist"
+                raise ConfigError, "in #{file}: source '#{source_name}' does not exist"
             end
 	end
 
@@ -743,6 +752,16 @@ module Autoproj
             each_remote_source(false).any? { true }
         end
 
+        # True if calling update_remote_sources will actually do anything
+        def should_update_remote_sources
+            each_remote_source(false) do |source|
+                if !File.directory?(source.local_dir)
+                    return true
+                end
+            end
+            false
+        end
+
         # Like #each_source, but filters out local package sets
         def each_remote_source(load_description = true)
             if !block_given?
@@ -1023,7 +1042,7 @@ module Autoproj
             else
                 source = each_source.find { |source| source.name == name }
                 if !source
-                    raise ConfigError, "#{name} is neither a package nor a source"
+                    raise ConfigError, "in #{file}: #{name} is neither a package nor a source"
                 end
                 packages.values.
                     find_all { |pkg| pkg.package_set.name == source.name }.
@@ -1104,6 +1123,13 @@ module Autoproj
             Autobuild::Package.each.map { |name, _| name }.to_set
         end
 
+        # Returns all the packages that can be built in this installation
+        def all_packages
+            packages.values.
+                map { |pkg| pkg.autobuild.name }.
+                find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
+        end
+
         # Returns the set of packages that should be built if the user does not
         # specify any on the command line
         def default_packages
@@ -1111,9 +1137,7 @@ module Autoproj
                         layout_packages(layout, true)
                     else
                         # No layout, all packages are selected
-                        packages.values.
-                            map { |pkg| pkg.autobuild.name }.
-                            find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
+                        all_packages
                     end
 
             names.delete_if { |pkg_name| excluded?(pkg_name) || ignored?(pkg_name) }
@@ -1156,7 +1180,7 @@ module Autoproj
                 begin
                     package.depends_on name
                 rescue Autobuild::ConfigException => e
-                    raise ConfigError, "manifest #{manifest_path} of #{package.name} from #{source.name} lists '#{name}' as dependency, which is listed in the layout but has no autobuild definition", e.backtrace
+                    raise ConfigError, "manifest #{manifest_path} of #{package.name} from #{source.name} lists '#{name}' as dependency, which is listed in the layout of #{file} but has no autobuild definition", e.backtrace
                 rescue ConfigError => e
                     raise ConfigError, "manifest #{manifest_path} of #{package.name} from #{source.name} lists '#{name}' as dependency, but it is neither a normal package nor an osdeps package. osdeps reports: #{e.message}", e.backtrace
                 end
@@ -1170,7 +1194,15 @@ module Autoproj
             selected_packages.each(&:load_package_manifest)
         end
 
-        def install_os_dependencies(packages)
+        # call-seq:
+        #   list_os_dependencies(packages) => required_packages, ospkg_to_pkg
+        #
+        # Returns the set of dependencies required by the listed packages.
+        #
+        # +required_packages+ is the set of osdeps names that are required for
+        # +packages+ and +ospkg_to_pkg+ a mapping from the osdeps name to the
+        # set of packages that require this OS package.
+        def list_os_dependencies(packages)
             required_os_packages = Set.new
             package_os_deps = Hash.new { |h, k| h[k] = Array.new }
             packages.each do |pkg_name|
@@ -1185,6 +1217,12 @@ module Autoproj
                 end
             end
 
+            return required_os_packages, package_os_deps
+        end
+
+        # Installs the OS dependencies that are required by the given packages
+        def install_os_dependencies(packages)
+            required_os_packages, package_os_deps = list_os_dependencies(packages)
             Autoproj.osdeps.install(required_os_packages, package_os_deps)
         end
 
@@ -1218,7 +1256,7 @@ module Autoproj
                     expanded_packages |= (packages & all_layout_packages)
                 end
 
-                !packages.empty? && !sources.empty?
+                !packages.empty? || !sources.empty?
             end
 
             if selected_packages.empty?
@@ -1237,21 +1275,17 @@ module Autoproj
             # Finally, check for package source directories
             all_packages = self.all_package_names
             selected_packages.delete_if do |sel|
-                match = Regexp.new("^#{Regexp.quote(sel)}")
+                match_dir      = Regexp.new("^#{Regexp.quote(sel)}")
+                match_pkg_name = Regexp.new(Regexp.quote(sel))
                 all_packages.each do |pkg_name|
                     pkg = Autobuild::Package[pkg_name]
-                    if pkg.srcdir =~ match
+                    if pkg_name =~ match_pkg_name || pkg.srcdir =~ match_dir
                         expanded_packages << pkg_name
                     end
                 end
             end
 
-            # Some packages might be there even though they are not listed in
-            # the layout (either un-defined or depended-upon by other packages).
-            # Check for those by looking for root_dir/package_name
-
-            # Finally, remove packages that are explicitely excluded and/or
-            # ignored
+            # Remove packages that are explicitely excluded and/or ignored
             expanded_packages.delete_if { |pkg_name| excluded?(pkg_name) || ignored?(pkg_name) }
             expanded_packages.to_set
         end