autoproj 1.5.8 → 1.6.0.rc1

data/History.txt

@@ -1,3 +1,8 @@
+= Version 1.6.0
+ * version that will be used in the Orocos project
+ * the osdeps management is now fully configurable. What is means is that you
+   can ask autoproj to *not* install osdeps packages at all
+
 = Version 1.5.8
  * add a way to add some ignore regexp to the dependency tracking. Updated user
    guide about that.

data/Rakefile

@@ -24,45 +24,11 @@ begin
                 ['webgen', '>= 0.5.9'] <<
                 ['rdoc', '>= 2.4.0']
         end
-
-        Rake.clear_tasks(/dist:publish_docs/)
-        task 'publish_docs' => 'doc' do
-            if !system('doc/update_github')
-                raise "cannot update the gh-pages branch for GitHub"
-            end
-            if !system('git', 'push', 'origin', '+gh-pages')
-                raise "cannot push the documentation"
-            end
-        end
-
-        desc "generate the bootstrap script"
-        task 'bootstrap' do
-            osdeps_code = File.read(File.join(Dir.pwd, 'lib', 'autoproj', 'osdeps.rb'))
-            osdeps_defaults = File.read(File.join(Dir.pwd, 'lib', 'autoproj', 'default.osdeps'))
-            # Filter rubygems dependencies from the OSdeps default. They will be
-            # installed at first build
-            osdeps = YAML.load(osdeps_defaults)
-            osdeps.delete_if do |name, content|
-                if content.respond_to?(:delete)
-                    content.delete('gem')
-                    content.empty?
-                else
-                    content == 'gem'
-                end
-            end
-            osdeps_defaults = YAML.dump(osdeps)
-
-            bootstrap_code = File.read(File.join(Dir.pwd, 'bin', 'autoproj_bootstrap.in')).
-                gsub('OSDEPS_CODE', osdeps_code).
-                gsub('OSDEPS_DEFAULTS', osdeps_defaults)
-            File.open(File.join(Dir.pwd, 'doc', 'guide', 'src', 'autoproj_bootstrap'), 'w') do |io|
-                io.write bootstrap_code
-            end
-        end
     end
 
     # Define our own documentation handling. Rake.clear_tasks is defined by Hoe
     Rake.clear_tasks(/dist:(re|clobber_|)docs/)
+    Rake.clear_tasks(/dist:publish_docs/)
 
 rescue LoadError
     STDERR.puts "cannot load the Hoe gem. Distribution is disabled"
@@ -71,6 +37,43 @@ rescue Exception => e
     STDERR.puts "error message is: #{e.message}"
 end
 
+namespace 'dist' do
+    task 'publish_docs' => 'doc' do
+        if !system('doc/update_github')
+            raise "cannot update the gh-pages branch for GitHub"
+        end
+        if !system('git', 'push', 'origin', '+gh-pages')
+            raise "cannot push the documentation"
+        end
+    end
+
+    desc "generate the bootstrap script"
+    task 'bootstrap' do
+        require 'yaml'
+        osdeps_code = File.read(File.join(Dir.pwd, 'lib', 'autoproj', 'osdeps.rb'))
+        osdeps_defaults = File.read(File.join(Dir.pwd, 'lib', 'autoproj', 'default.osdeps'))
+        # Filter rubygems dependencies from the OSdeps default. They will be
+        # installed at first build
+        osdeps = YAML.load(osdeps_defaults)
+        osdeps.delete_if do |name, content|
+            if content.respond_to?(:delete)
+                content.delete('gem')
+                content.empty?
+            else
+                content == 'gem'
+            end
+        end
+        osdeps_defaults = YAML.dump(osdeps)
+
+        bootstrap_code = File.read(File.join(Dir.pwd, 'bin', 'autoproj_bootstrap.in')).
+            gsub('OSDEPS_CODE', osdeps_code).
+            gsub('OSDEPS_DEFAULTS', osdeps_defaults)
+        File.open(File.join(Dir.pwd, 'doc', 'guide', 'src', 'autoproj_bootstrap'), 'w') do |io|
+            io.write bootstrap_code
+        end
+    end
+end
+
 do_doc = begin
              require 'webgen/webgentask'
              require 'rdoc/task'

data/bin/autoproj

@@ -28,7 +28,7 @@ def report(debug)
 rescue ConfigError => e
     STDERR.puts
     STDERR.puts color(e.message, :red, :bold)
-    root_dir = /^#{Regexp.quote(Autoproj.root_dir)}/
+    root_dir = /^#{Regexp.quote(Autoproj.root_dir)}(?!\/\.gems)/
     e.backtrace.find_all { |path| path =~ root_dir }.
         each do |path|
             STDERR.puts color("  in #{path}", :red, :bold)

data/doc/guide/src/autoproj_bootstrap

@@ -27,14 +27,25 @@ module Autoproj
     class ConfigError < RuntimeError; end
     class << self
         attr_reader :verbose
+        attr_reader :config
     end
+    @config = Hash.new
 
     # Fake option management for the OSdeps class
-    def self.has_config_key?(*args)
-        false
+    def self.has_config_key?(key)
+        config.has_key?(key)
     end
-    def self.change_option(*args)
-        false
+
+    def self.user_config(key)
+        config[key]
+    end
+
+    def self.change_option(name, value, ignored = true)
+        config[name] = value
+    end
+
+    def self.color(string, *args)
+        string
     end
 end
 
@@ -90,6 +101,8 @@ module Autoproj
 
         class << self
             attr_reader :aliases
+            attr_accessor :force_osdeps
+            attr_accessor :gem_with_prerelease
         end
         @aliases = Hash.new
 
@@ -217,11 +230,13 @@ module Autoproj
                         ['gentoo', [version]]
                     elsif File.exists?('/etc/arch-release')
                         ['arch', []]
-                    else
-                        raise ConfigError, "Unknown operating system"
                     end
             end
 
+            if !@operating_system
+                return
+            end
+
             # Normalize the names to lowercase
             @operating_system =
                 [@operating_system[0].downcase,
@@ -257,62 +272,118 @@ module Autoproj
             'arch' => 'pacman -Sy --noconfirm %s'
         }
 
-        # Resolves the given OS dependencies into the actual packages that need
-        # to be installed on this particular OS.
+        NO_PACKAGE       = 0
+        WRONG_OS         = 1
+        WRONG_OS_VERSION = 2
+        IGNORE           = 3
+        PACKAGES         = 4
+        SHELL_SNIPPET    = 5
+        UNKNOWN_OS       = 7
+        AVAILABLE        = 10
+
+        # Check for the definition of +name+ for this operating system
         #
-        # Raises ConfigError if some packages can't be found
-        def resolve_os_dependencies(dependencies)
+        # It can return
+        #
+        # NO_PACKAGE::
+        #   there are no package definition for +name
+        # UNKNOWN_OS::
+        #   this is not an OS autoproj knows how to deal with
+        # WRONG_OS::
+        #   there are a package definition, but not for this OS
+        # WRONG_OS_VERSION::
+        #   there is a package definition for this OS, but not for this
+        #   particular version of the OS
+        # IGNORE::
+        #   there is a package definition that told us to ignore the package
+        # [PACKAGES, definition]::
+        #   +definition+ is an array of package names that this OS's package
+        #   manager can understand
+        # [SHELL_SNIPPET, definition]::
+        #   +definition+ is a string which is a shell snippet that will install
+        #   the package
+        def resolve_package(name)
             os_name, os_version = OSDependencies.operating_system
-            if !OS_PACKAGE_INSTALL.has_key?(os_name)
-                raise ConfigError, "I don't know how to install packages on #{os_name}"
+
+            dep_def = definitions[name]
+            if !dep_def
+                return NO_PACKAGE
             end
 
-            os_packages    = []
-            shell_snippets = []
-            dependencies.each do |name|
-                dep_def = definitions[name]
-                if !dep_def
-                    raise ConfigError, "I don't know how to install '#{name}'"
-                end
+            if !os_name
+                return UNKNOWN_OS
+            end
+
+            # Find a matching entry for the OS name
+            os_entry = dep_def.find do |name_list, data|
+                name_list.split(',').
+                    map(&:downcase).
+                    any? { |n| n == os_name }
+            end
+
+            if !os_entry
+                return WRONG_OS
+            end
+
+            data = os_entry.last
+
+            # This package does not need to be installed on this operating system (example: build tools on Gentoo)
+            if !data || data == "ignore"
+                return IGNORE
+            end
 
-                # Find a matching entry for the OS name
-                os_entry = dep_def.find do |name_list, data|
-                    name_list.split(',').
+            if data.kind_of?(Hash)
+                version_entry = data.find do |version_list, data|
+                    version_list.to_s.split(',').
                         map(&:downcase).
-                        any? { |n| n == os_name }
+                        any? do |v|
+                        os_version.any? { |osv| Regexp.new(v) =~ osv }
+                        end
                 end
 
-                if !os_entry
-                    raise ConfigError, "I don't know how to install '#{name}' on #{os_name}"
+                if !version_entry
+                    return WRONG_OS_VERSION
                 end
+                data = version_entry.last
+            end
 
-                data = os_entry.last
-
-                # This package does not need to be installed on this operating system (example: build tools on Gentoo)
-                next if !data || data == "ignore"
+            if data.respond_to?(:to_ary)
+                return [PACKAGES, data]
+            elsif data.to_str =~ /\w+/
+                return [PACKAGES, [data.to_str]]
+            else
+                return [SHELL_SNIPPET, data.to_str]
+            end
+        end
 
-                if data.kind_of?(Hash)
-                    version_entry = data.find do |version_list, data|
-                        version_list.to_s.split(',').
-                            map(&:downcase).
-                            any? do |v|
-                                os_version.any? { |osv| Regexp.new(v) =~ osv }
-                            end
-                    end
+        # Resolves the given OS dependencies into the actual packages that need
+        # to be installed on this particular OS.
+        #
+        # Raises ConfigError if some packages can't be found
+        def resolve_os_dependencies(dependencies)
+            os_name, os_version = OSDependencies.operating_system
 
-                    if !version_entry
-                        raise ConfigError, "I don't know how to install '#{name}' on this specific version of #{os_name} (#{os_version.join(", ")})"
-                    end
-                    data = version_entry.last
+            os_packages    = []
+            shell_snippets = []
+            dependencies.each do |name|
+                result = resolve_package(name)
+                if result == NO_PACKAGE
+                    raise ConfigError, "there is no osdeps definition for #{name}"
+                elsif result == WRONG_OS
+                    raise ConfigError, "there is an osdeps definition for #{name}, but not for this operating system"
+                elsif result == WRONG_OS_VERSION
+                    raise ConfigError, "there is an osdeps definition for #{name}, but no for this particular operating system version"
+                elsif result == IGNORE
+                    next
+                elsif result[0] == PACKAGES
+                    os_packages.concat(result[1])
+                elsif result[0] == SHELL_SNIPPET
+                    shell_snippets << result[1]
                 end
+            end
 
-                if data.respond_to?(:to_ary)
-                    os_packages.concat data.to_ary
-                elsif data.to_str =~ /\w+/
-                    os_packages << data.to_str
-                else
-                    shell_snippets << data.to_str
-                end
+            if !OS_PACKAGE_INSTALL.has_key?(os_name)
+                raise ConfigError, "I don't know how to install packages on #{os_name}"
             end
 
             return os_packages, shell_snippets
@@ -329,14 +400,27 @@ module Autoproj
                 "\n" + shell_snippets.join("\n")
         end
 
-        # Returns true if there is an operating-system package with that name,
-        # and false otherwise
+        # Returns true if +name+ is an acceptable OS package for this OS and
+        # version
         def has?(name)
+            availability_of(name) == AVAILABLE
+        end
+
+        # If +name+ is an osdeps that is available for this operating system,
+        # returns AVAILABLE. Otherwise, returns the same error code than
+        # resolve_package.
+        def availability_of(name)
             osdeps, gemdeps = partition_packages([name].to_set)
-            resolve_os_dependencies(osdeps)
-            true
-        rescue ConfigError
-            false
+            if !osdeps.empty?
+                status = resolve_package(name)
+                if status.respond_to?(:to_ary) || status == IGNORE
+                    AVAILABLE
+                else
+                    status
+                end
+            else
+                AVAILABLE
+            end
         end
 
         # call-seq:
@@ -357,12 +441,10 @@ module Autoproj
             package_set.to_set.each do |name|
                 pkg_def = definitions[name]
                 if !pkg_def
-                    msg = "I know nothing about a prepackaged software called '#{name}'"
-                    if pkg_names = package_osdeps[name]
-                        msg += ", it is listed as dependency of the following package(s): #{pkg_names.join(", ")}"
-                    end
-
-                    raise ConfigError, msg
+                    # Error cases are taken care of later, because that is were
+                    # the automatic/manual osdeps logic lies
+                    osdeps << name
+                    next
                 end
 
                 pkg_def = pkg_def.dup
@@ -373,6 +455,9 @@ module Autoproj
                     when "gem" then
                         gems << name
                     else
+                        # This is *not* handled later, as is the absence of a
+                        # package definition. The reason is that it is a bad
+                        # configuration file, and should be fixed by the user
                         raise ConfigError, "unknown OS-independent package management type #{pkg_def} for #{name}"
                     end
                 else
@@ -403,19 +488,123 @@ module Autoproj
             end
         end
 
+        def filter_uptodate_gems(gems)
+            Autobuild.progress "looking for RubyGems updates"
+
+            # Don't install gems that are already there ...
+            gems = gems.dup
+            gems.delete_if do |name|
+                version_requirements = Gem::Requirement.default
+                installed = Gem.source_index.find_name(name, version_requirements)
+                if !installed.empty? && Autobuild.do_update
+                    # Look if we can update the package ...
+                    dep = Gem::Dependency.new(name, version_requirements)
+                    available = gem_fetcher.find_matching(dep)
+                    installed_version = installed.map(&:version).max
+                    available_version = available.map { |(name, v), source| v }.max
+                    needs_update = (available_version > installed_version)
+                    !needs_update
+                else
+                    !installed.empty?
+                end
+            end
+            gems
+        end
+
+        AUTOMATIC = true
+        MANUAL    = false
+        WAIT      = :wait
+        ASK       = :ask
+
+        def automatic_osdeps_mode
+            if mode = ENV['AUTOPROJ_AUTOMATIC_OSDEPS']
+                mode =
+                    if mode == 'true' then AUTOMATIC
+                    elsif mode == 'false' then MANUAL
+                    elsif mode == 'wait' then WAIT
+                    else ASK
+                    end
+                Autoproj.change_option('automatic_osdeps', mode, true)
+                mode
+            else
+                Autoproj.user_config('automatic_osdeps')
+            end
+        end
+
         # Requests the installation of the given set of packages
         def install(packages, package_osdeps = Hash.new)
+            os_def = OSDependencies.operating_system
             osdeps, gems = partition_packages(packages, package_osdeps)
+            gems = filter_uptodate_gems(gems)
+            if osdeps.empty? && gems.empty?
+                return
+            end
+
+            if automatic_osdeps_mode == AUTOMATIC && !os_def && !osdeps.empty?
+                puts
+                puts Autoproj.color("==============================", :bold)
+                puts Autoproj.color("The packages that will be built require some other software to be installed", :bold)
+                puts "  " + osdeps.join("\n  ")
+                puts Autoproj.color("==============================", :bold)
+                puts
+            end
+
+            if !OSDependencies.force_osdeps && automatic_osdeps_mode != AUTOMATIC
+                puts
+                puts Autoproj.color("==============================", :bold)
+                puts Autoproj.color("The packages that will be built require some other software to be installed", :bold)
+                puts
+                if !osdeps.empty?
+                    puts "From the operating system:"
+                    puts "  " + osdeps.join("\n  ")
+                    puts
+                end
+                if !gems.empty?
+                    puts "From RubyGems:"
+                    puts "  " + gems.join("\n  ")
+                    puts
+                end
+
+                if automatic_osdeps_mode == ASK
+                    print "Should I install these packages ? [yes] "
+                    STDOUT.flush
+                    do_osdeps = nil
+                    while do_osdeps.nil?
+                        answer = STDIN.readline.chomp
+                        if answer == ''
+                            do_osdeps = true
+                        elsif answer == "no"
+                            do_osdeps = false
+                        elsif answer == 'yes'
+                            do_osdeps = true
+                        else
+                            print "invalid answer. Please answer with 'yes' or 'no' "
+                            STDOUT.flush
+                        end
+                    end
+                else
+                    puts "Since you requested autoproj to not handle the osdeps automatically, you have to"
+                    puts "do it yourself. Alternatively, you can run 'autoproj osdeps' and/or change to"
+                    puts "automatic osdeps handling by running an autoproj operation with the --reconfigure"
+                    puts "option (e.g. autoproj build --reconfigure)"
+                    puts Autoproj.color("==============================", :bold)
+                    puts
+
+                    if automatic_osdeps_mode == WAIT
+                        print "Press ENTER to continue "
+                        STDOUT.flush
+                        STDIN.readline
+                    end
+                end
+
+                if !do_osdeps
+                    return
+                end
+            end
 
             did_something = false
 
-            # Ideally, we would feed the OS dependencies to rosdep.
-            # Unfortunately, this is C++ code and I don't want to install the
-            # whole ROS stack just for rosdep ...
-            #
-            # So, for now, reimplement rosdep by ourselves. Given how things
-            # are done, this is actually not so hard.
-            if !osdeps.empty?
+            if os_def && !osdeps.empty?
                 shell_script = generate_os_script(osdeps)
                 if Autoproj.verbose
                     Autoproj.progress "Installing non-ruby OS dependencies with"
@@ -435,35 +624,25 @@ module Autoproj
             end
 
             if !gems.empty?
-                Autobuild.progress "looking for RubyGems updates"
-                # Don't install gems that are already there ...
-                gems.delete_if do |name|
-                    version_requirements = Gem::Requirement.default
-                    installed = Gem.source_index.find_name(name, version_requirements)
-                    if !installed.empty? && Autobuild.do_update
-                        # Look if we can update the package ...
-                        dep = Gem::Dependency.new(name, version_requirements)
-                        available = gem_fetcher.find_matching(dep)
-                        installed_version = installed.map(&:version).max
-                        available_version = available.map { |(name, v), source| v }.max
-                        needs_update = (available_version > installed_version)
-                        !needs_update
-                    else
-                        !installed.empty?
-                    end
-                end
+                gems = filter_uptodate_gems(gems)
             end
 
             # Now install what is left
             if !gems.empty?
                 guess_gem_program
-
                 if Autoproj.verbose
                     Autoproj.progress "Installing rubygems dependencies with"
                     Autoproj.progress "gem install #{gems.join(" ")}"
                 end
+
+                cmdline = [Autobuild.tool('gem'), 'install']
+                if Autobuild::OSDependencies.gem_with_prerelease
+                    cmdline << "--prerelease"
+                end
+                cmdline.concat(gems)
+
                 Autobuild.progress "installing/updating RubyGems dependencies: #{gems.join(", ")}"
-                Autobuild::Subprocess.run 'autoproj', 'osdeps', Autobuild.tool('gem'), 'install', *gems
+                Autobuild::Subprocess.run 'autoproj', 'osdeps', *cmdline
                 did_something ||= true
             end
 
@@ -560,6 +739,69 @@ rdoc: gem
 
 EODEFS
 
+operating_system = Autoproj::OSDependencies.operating_system
+if !operating_system
+    puts <<-EOT
+
+autoproj is usually able to handle the installation of operating system packages
+himself. However, it does not know about your operating system, and will
+therefore have to let you install the packages yourself.
+
+It can still install the RubyGems packages that are required by the built
+packages.
+
+If you answer 'no' to the following question, autoproj will always go on,
+assuming that all the required dependencies are already installed. If you answer
+'yes', it will make sure that the RubyGems are installed himself, but it will
+assume that you took care of the other packages. If you answer 'wait', it will
+not install anything but will wait for you to press ENTER each time some
+external software is needed. Finally, if you answer 'ask', it will ask you each
+time what it should do when there is something to install
+
+    EOT
+
+    print "How should autoproj install RubyGems packages automatically (yes, no, wait or ask) ? [yes] "
+    STDOUT.flush
+else
+    # Before doing *anything*, we have to ask the user if he wants us to install
+    # the osdeps ...
+    puts <<-EOT
+
+autoproj is able to handle the installation of operating system packages
+himself. This functionality will use 'sudo' to gain root access each time
+it is needed.
+
+If you answer 'no' to the following question, autoproj will go on, assuming that
+all the required dependencies are already installed. If you answer 'yes', it
+will make sure that they are installed himself. If you answer 'wait', it will
+not install anything but wait for you to press ENTER each time some external
+software is needed.  Finally, if you answer 'ask', it will ask you what it
+should do each time there is something to install.
+
+    EOT
+    print "Should autoproj install OS packages automatically (yes, no, wait or ask) ? [yes] "
+    STDOUT.flush
+end
+
+automatic_osdeps = nil
+while automatic_osdeps.nil?
+    answer = STDIN.readline.chomp
+    if answer == ''
+        automatic_osdeps = true
+    elsif answer == "no"
+        automatic_osdeps = false
+    elsif answer == 'ask'
+        automatic_osdeps = :ask
+    elsif answer == 'yes'
+        automatic_osdeps = true
+    else
+        print "invalid answer. Please answer with 'yes', 'no' or 'ask' "
+        STDOUT.flush
+    end
+end
+Autoproj.config['automatic_osdeps'] =
+    automatic_osdeps
+
 # First thing we do is install a proper ruby environment. We make sure that we
 # aren't installing any gems for now (as we need to choose the right gem
 # binary) by setting Autobuild.programs['gem'] to nil
@@ -582,10 +824,7 @@ USER_PACKAGES = %w{autoproj}
 ENV['RUBYOPT']  = "-rubygems"
 require 'rubygems'
 
-packages = PACKAGES.dup
-if ARGV.first != "dev"
-    packages += USER_PACKAGES
-end
+packages = PACKAGES + USER_PACKAGES
 
 begin
     osdeps_management = Autoproj::OSDependencies.new(YAML.load(DEFS))
@@ -596,8 +835,14 @@ rescue Autoproj::ConfigError => e
     exit(1)
 end
 
-if ARGV.first != "dev"
-    Autobuild::Subprocess.run('bootstrap', 'post', 'autoproj', 'bootstrap', *ARGV)
+if ARGV.first == "dev"
+    Autoproj::OSDependencies.gem_with_prerelease = true
+end
+
+ENV['AUTOPROJ_AUTOMATIC_OSDEPS'] = automatic_osdeps.to_s
+if !system('autoproj', 'bootstrap', *ARGV)
+    STDERR.puts "ERROR: failed to run autoproj bootstrap #{ARGV.join(", ")}"
+    exit 1
 end
 
 File.open('env.sh', 'w') do |io|