autoproj 1.2.6 → 1.3.0

data/History.txt

@@ -1,3 +1,7 @@
+= Version 1.3.0
+ * Fix limitations of using layouts. autoproj is now able to find out
+   cross-layout dependencies and act accordingly.
+ 
 = Version 1.2.6
  * Fix mail reporting
 

data/Manifest.txt

@@ -12,6 +12,7 @@ doc/guide/src/autoproj_bootstrap
 doc/guide/src/customization.page
 doc/guide/src/default.css
 doc/guide/src/default.template
+doc/guide/src/error_messages.page
 doc/guide/src/htmldoc.metainfo
 doc/guide/src/htmldoc.virtual
 doc/guide/src/images/bodybg.png

data/README.txt

@@ -32,14 +32,16 @@ Components of an Autoproj installation
 -------------------------------------
 A autoproj installation is seeded by _package sets_. A package set is a local or remote
 directory in which there is:
- * autobuild scripts that describe what can be built and how it should be built.
-   These scripts an also list a set of configuration options that allow to
-   parametrize the build. In general, there should be only a limited number of
-   such options.
- * a source.yml file which describes the package set itself, and where the software
-   packages are located (what version control system and what URL).
- * optionally, a file that describe prepackaged dependencies that can be
-   installed by using the operating system package management system.
+
+ * a list of available packages, and information on how to build these packages.
+   This list, and the build information, are Ruby scripts that use the autobuild
+   API. [More ...](autobuild.html)
+ * version control information: where to get the source code for each of the
+   packages defined in the set.
+
+Then, an autoproj configuration is simply a list of package sets (i.e. a list of
+packages that are available to build), and a file that selects which packages
+should be built. This file is the <tt>manifest</tt>.
 
 Bootstrapping
 -------------
@@ -59,7 +61,28 @@ The canonical way is the following:
    ruby autoproj\_bootstrap
    {.cmdline}
 
- * follow the instructions printed by the script above :)
+ * follow the instructions printed by the script<tt>manifest</tt>.
+
+Additionally, if you are given a reference to a source code repository in which
+an autoproj configuration is stored (i.e. a directory in which a manifest is
+present), you can bootstrap this configuration directly:
+
+   wget http://doudou.github.com/autoproj/autoproj\_bootstrap <br />
+   ruby autoproj\_bootstrap VCS 
+   {.cmdline}
+
+For instance, to build all packages made available by the RubyInMotion project,
+do
+
+   wget http://doudou.github.com/autoproj/autoproj\_bootstrap <br />
+   ruby autoproj\_bootstrap git git://github.com/doudou/rubim.all.git
+   {.cmdline}
+
+Additional options can be given for the version control system. For instance,
+
+   wget http://doudou.github.com/autoproj/autoproj\_bootstrap <br />
+   ruby autoproj\_bootstrap git git://github.com/doudou/rubim.all.git branch=stable
+   {.cmdline}
 
 Software packages in Autoproj
 -----------------------------

data/bin/autoproj

@@ -113,28 +113,27 @@ args = ARGV.dup
 parser.parse!(args)
 mode     = args.shift
 selected_packages = args.dup
-all_env_sh = Array.new
 Autobuild::Reporting << Autoproj::Reporter.new
 
 def color(*args)
     Autoproj.console.color(*args)
 end
 
-def do_status(packages)
-    console = Autoproj.console
+def display_status(packages)
+    last_was_in_sync = false
 
-    Autobuild::Reporting.report do
-        last_was_in_sync = false
-        packages.sort.each do |pkg|
-            pkg = Autobuild::Package[pkg]
+    packages.each do |pkg|
+        lines = []
 
-            if !pkg.importer.respond_to?(:status)
-                # This importer does not support status display
-                STDERR.puts color("  this package's importer does not support status display", :bold, :red)
-                next
-            end
+        if !pkg.importer.respond_to?(:status)
+            # This importer does not support status display
+            STDERR.puts color("  the importer of #{pkg.autoproj_name} does not support status display (#{pkg.importer.class.name})", :bold, :red)
+            next
+        end
 
-            lines = []
+        if !File.directory?(pkg.srcdir)
+            lines << color("  is not imported yet", :magenta)
+        else
             status = pkg.importer.status(pkg)
             if status.uncommitted_code
                 lines << color("  contains uncommitted modifications", :red)
@@ -144,9 +143,9 @@ def do_status(packages)
             when Autobuild::Importer::Status::UP_TO_DATE
                 if !status.uncommitted_code
                     if last_was_in_sync
-                        STDERR.print ", #{pkg.name}"
+                        STDERR.print ", #{pkg.autoproj_name}"
                     else
-                        STDERR.print pkg.name
+                        STDERR.print pkg.autoproj_name
                     end
                     last_was_in_sync = true
                     next
@@ -174,25 +173,51 @@ def do_status(packages)
                     lines << color("   #{line}", :magenta)
                 end
             end
+        end
 
-            if last_was_in_sync
-                STDERR.puts color(": local and remote are in sync", :green)
-            end
+        if last_was_in_sync
+            STDERR.puts color(": local and remote are in sync", :green)
+        end
 
-            last_was_in_sync = false
-            STDERR.print "#{pkg.name}:"
+        last_was_in_sync = false
+        if pkg.respond_to?(:text_name)
+            STDERR.print "#{pkg.text_name}:"
+        else
+            STDERR.print "#{pkg.autoproj_name}:"
+        end
 
-            if lines.size == 1
-                STDERR.puts lines.first
-            else
-                STDERR.puts
-                STDERR.puts lines.join("\n")
-            end
+        if lines.size == 1
+            STDERR.puts lines.first
+        else
+            STDERR.puts
+            STDERR.puts lines.join("\n")
         end
-        if last_was_in_sync
-            STDERR.puts color(": local and remote are in sync", :green)
+    end
+    if last_was_in_sync
+        STDERR.puts color(": local and remote are in sync", :green)
+    end
+end
+def do_status(packages)
+    console = Autoproj.console
+    
+    sources = Autoproj.manifest.each_configuration_source.
+        map do |vcs, text_name, pkg_name, local_dir|
+            Autoproj::Manifest.create_autobuild_package(vcs, text_name, pkg_name, local_dir)
         end
+
+    if !sources.empty?
+        STDERR.puts color("autoproj: displaying status of configuration", :bold)
+        display_status(sources)
+        STDERR.puts
     end
+
+
+    STDERR.puts color("autoproj: displaying status of packages", :bold)
+    packages = packages.sort.map do |pkg_name|
+        Autobuild::Package[pkg_name]
+    end
+    display_status(packages)
+
 end
 
 def switch_config(*args)
@@ -241,11 +266,9 @@ def do_switch_config(*args)
     vcs = Autoproj.normalize_vcs_definition(vcs_def)
 
     # Install the OS dependencies required for this VCS
-    Autobuild::Reporting.report do
-        osdeps = Autoproj::OSDependencies.load_default
-        osdeps.install([vcs.type])
-        Autoproj::Manifest.import_whole_installation(vcs, File.join(Dir.pwd, "autoproj"))
-    end
+    osdeps = Autoproj::OSDependencies.load_default
+    osdeps.install([vcs.type])
+    Autoproj::Manifest.update_source(vcs, "autoproj main configuration", File.join(Dir.pwd, "autoproj"))
 
     # Now write it in the config file
     File.open(File.join(Autoproj.config_dir, "config.yml"), "a") do |io|
@@ -320,8 +343,31 @@ EOTEXT
 
 end
 
+def report(debug)
+    Autobuild::Reporting.report do
+        yield
+        Autobuild::Reporting.success
+    end
+
+rescue ConfigError => e
+    STDERR.puts
+    STDERR.puts color(e.message, :red, :bold)
+    if debug then raise
+    else exit 1
+    end
+rescue Interrupt
+    STDERR.puts
+    STDERR.puts color("Interrupted by user", :red, :bold)
+    if debug then raise
+    else exit 1
+    end
+end
+
+Autobuild.doc_errors = false
+Autobuild.do_doc = false
+
 # Find the autoproj root dir
-begin
+report(debug) do
     case mode
     when "bootstrap"
         # If there is no argument, We need one more argument. It is either a VCS type or a path to a
@@ -398,8 +444,6 @@ begin
     Autobuild.prefix  = Autoproj.build_dir
     Autobuild.srcdir  = root_dir
     Autobuild.logdir = File.join(Autobuild.prefix, 'log')
-    Autobuild.doc_errors = false
-    Autobuild.do_doc = false
     if mail_config[:to]
         Autobuild::Reporting << Autobuild::MailReporter.new(mail_config)
     end
@@ -417,9 +461,7 @@ begin
     # installed (i.e. that the user did not remove it)
     manifest = Manifest.load(manifest_path)
     if manifest.vcs && mode != "bootstrap"
-        Autobuild::Reporting.report do
-            manifest.update_yourself
-        end
+        manifest.update_yourself
         manifest = Manifest.load(manifest_path)
     end
     Autoproj.manifest = manifest
@@ -432,18 +474,14 @@ begin
     # If we need to install some packages to import our remote sources, do it
     if !no_os_deps && !source_os_dependencies.empty?
         STDERR.puts color("autoproj: installing prepackaged dependencies to access the source definitions", :bold)
-        Autobuild::Reporting.report do
-            osdeps = manifest.known_os_packages
-            osdeps.install(source_os_dependencies)
-        end
+        osdeps = manifest.known_os_packages
+        osdeps.install(source_os_dependencies)
     end
 
     # Update the remote sources if there are any
     if manifest.has_remote_sources?
         STDERR.puts color("autoproj: updating remote definitions of package sets", :bold)
-        Autobuild::Reporting.report do
-            manifest.update_remote_sources
-        end
+        manifest.update_remote_sources
         STDERR.puts
     end
 
@@ -502,16 +540,34 @@ begin
                 if source.local?
                     STDERR.puts "    local source in #{source.local_dir}"
                 else
-                    STDERR.puts "    vcs:   #{source.vcs}, #{source.vcs.options.inspect}"
+                    STDERR.puts "    from:  #{source.vcs}"
                     STDERR.puts "    local: #{source.local_dir}"
                 end
 
+                lines = []
                 source.each_package.
-                    map { |pkg| pkg.name }.
-                    sort.
-                    each do |name|
-                        STDERR.puts "    #{name}"
+                    map { |pkg| [pkg.name, Autoproj.manifest.package_manifests[pkg.name]] }.
+                    sort_by { |name, _| name }.
+                    each do |name, manifest|
+                        vcs_def = Autoproj.manifest.importer_definition_for(name)
+                        if manifest
+                            lines << [name, manifest.short_documentation]
+                            lines << ["", vcs_def.to_s]
+                        else
+                            lines << [name, vcs_def.to_s]
+                        end
                     end
+
+                w_col1, w_col2 = nil
+                lines.each do |col1, col2|
+                    w_col1 = col1.size if !w_col1 || col1.size > w_col1
+                    w_col2 = col2.size if !w_col2 || col2.size > w_col2
+                end
+                STDERR.puts "    packages:"
+                format = "    | %-#{w_col1}s | %-#{w_col2}s |"
+                lines.each do |col1, col2|
+                    STDERR.puts(format % [col1, col2])
+                end
             end
         end
     end
@@ -538,10 +594,8 @@ begin
     if !no_os_deps
         STDERR.puts
         STDERR.puts color("autoproj: installing prepackaged dependencies for build system & version control", :bold)
-        Autobuild::Reporting.report do
-            osdeps = manifest.known_os_packages
-            osdeps.install(Autoproj.build_system_dependencies - source_os_dependencies)
-        end
+        osdeps = manifest.known_os_packages
+        osdeps.install(Autoproj.build_system_dependencies - source_os_dependencies)
     end
 
     # Now starts a different stage of the whole build. Until now, we were
@@ -551,23 +605,10 @@ begin
     # First, we allow to user to specify packages based on disk paths, so
     # resolve those
     selected_packages = Autoproj.manifest.expand_package_selection(selected_packages)
+    seen = Set.new
+    Autoproj.manifest.each_package_set do |name, packages, enabled_packages|
+        packages -= seen
 
-    # This keeps a hash of package_name => layout_name for all packages that
-    # have already been handled
-    handled_packages = Hash.new
-
-    Autoproj.manifest.each_package_set(selected_packages) do |name, packages, enabled_packages|
-        packages         -= handled_packages.keys.to_set
-        enabled_packages -= handled_packages.keys.to_set
-
-        # Here, 'packages' are the packages listed in the layout configuration,
-        # and 'enabled_packages' the ones that 
-        #
-        # They do not (yet) take into account dependency information -- this is
-        # not doable as the manifests have not been loaded yet (the packages
-        # aren't imported). We add this information later on
-
-        # Setup directories
         srcdir  = File.join(Autoproj.root_dir, name)
         prefix  = File.join(Autoproj.build_dir, name)
         logdir  = File.join(prefix, "log")
@@ -578,134 +619,142 @@ begin
             pkg.prefix = prefix
             pkg.logdir = logdir
         end
+        seen |= packages
+    end
 
-        # We are doing a status, now is the right time. Otherwise, build ! But
-        # only if there is some packages to build (and avoid display of progress
-        # messages if there is nothing to build)
-        all_enabled_packages  = Set.new
+    if only_do_status
         STDERR.puts
-        if only_do_status
-            STDERR.puts color("autoproj: status of packages in #{name}", :bold)
-            do_status(enabled_packages)
-        elsif !enabled_packages.empty?
-            Autobuild::Reporting.report do
-                STDERR.puts
-                STDERR.puts color("autoproj: now building #{name}", :bold)
-
-                STDERR.puts color("  updating packages", :bold)
-                # We actually have a problem there: the packages in
-                # enabled_packages are *not* taking into account dependencies as
-                # we can't load the manifests yet ...
-                #
-                # So we have to make our own dependency analysis. Fortunately
-                # (for us), there is no need to import packages in order, so we
-                # can do a simple BFS
-                #
-                # Additional benefit: it computes the set of packages 
-                packages_to_import = enabled_packages.dup
-                while !packages_to_import.empty?
-                    import_now, packages_to_import = packages_to_import, Set.new
-                    import_now.each do |pkg_name|
-                        # Already handled at another place in the layout
-                        next if handled_packages.has_key?(pkg_name)
-                        # Already handled in this part of the layout
-                        next if all_enabled_packages.include?(pkg_name)
-                        # Not handled already. Import, load the manifest, add to
-                        # all_enabled_packages and add the dependencies
-                        Rake::Task["#{pkg_name}-import"].invoke
-                        manifest.load_package_manifest(pkg_name)
-                        all_enabled_packages << pkg_name
-                        Autobuild::Package[pkg_name].dependencies.each do |dep_name|
-                            if !handled_packages.has_key?(dep_name) &&
-                                !packages.include?(dep_name)
-                                raise ConfigError, "#{pkg_name}, at #{name} in the layout, depends on #{dep_name} but this package appears in the layout neither at #{name} nor before"
-                            end
-                            packages_to_import << dep_name
-                        end
-                    end
-                end
-
-                if !no_os_deps
-                    STDERR.puts color("  installing prepackaged dependencies", :bold)
-                    manifest.install_os_dependencies(all_enabled_packages)
-                end
+        all_packages = Set.new
+        Autoproj.manifest.handle_enabled_packages(selected_packages) do |name, _, layout_enabled|
+            all_packages |= layout_enabled
+        end
+        do_status(all_packages)
+        exit(0)
+    end
 
-                # And now build
-                if Autobuild.only_doc
-                    STDERR.puts color("  building and installing documentation", :bold)
-                else
-                    STDERR.puts color("  building and installing packages", :bold)
-                end
 
-                Autobuild.apply(all_enabled_packages, "autoproj-#{name}")
-                Autobuild::Reporting.success
-            end
-        end
-        
-        # Now mark them as handled
-        packages.each do |pkg_name|
-            handled_packages[pkg_name] = name
-        end
+    # First thing: do the import. We are handling the imports ourselves as it
+    # allows us to complete the enabled_packages set with the package
+    # dependencies. The scheme is as follows:
+    #  * we import a package, prepare it and load its manifest
+    #  * we look at its dependencies. Dependencies are either already handled
+    #    (and nothing needs to be done) or are to-be-handled at this level.
+    #    Otherwise, it is considered as an error
+    #
+    seen = Set.new
+    Autoproj.manifest.each_package_set do |name, packages, enabled_packages|
+        packages -= seen
 
-        # Now call the prepare target for all packages as it may be useful for
-        # the rest of the builds and for the generation of the env.sh file
-        #
-        # Note that we don't really have to care about dependencies, but we do
-        # it anyway
         old_update_flag = Autobuild.do_update
         begin
             Autobuild.do_update = false
-            Autobuild::Reporting.report do
-                prepare_targets = (packages - enabled_packages).
-                    find_all { |pkg_name| File.directory?(Autobuild::Package[pkg_name].srcdir) }.
-                    map { |pkg_name| "#{pkg_name}-prepare" }
-                task "autoproj-#{name}-prepare" => prepare_targets
-                Rake::Task["autoproj-#{name}-prepare"].invoke
+            packages.each do |pkg_name|
+                pkg = Autobuild::Package[pkg_name]
+                if File.directory?(pkg.srcdir)
+                    Rake::Task["#{pkg_name}-import"].invoke
+                end
             end
         ensure
             Autobuild.do_update = old_update_flag
         end
 
-        if (mode == "build" || mode == "fast-build")
-            if packages.all? { |pkg_name| File.directory?(Autobuild::Package[pkg_name].srcdir) }
-                all_env_sh << name
-                Autoproj.export_env_sh(name)
-            else
-                STDERR.puts color("WARN: #{name} has not been completely built, #{name}env.sh is not updated", :magenta)
+        seen |= packages
+    end
+
+    STDERR.puts
+    STDERR.puts color("autoproj: importing and loading selected packages", :bold)
+    all_packages         = Set.new
+    all_enabled_packages = Set.new
+    all_sublayouts       = Set.new
+    Autoproj.manifest.handle_enabled_packages(selected_packages) do |name, packages, enabled_packages, _|
+        packages         -= all_enabled_packages
+        enabled_packages -= all_enabled_packages
+        all_sublayouts << name
+
+        packages_to_import = enabled_packages.dup.to_set
+        while !packages_to_import.empty?
+            import_now, packages_to_import = packages_to_import, Set.new
+            import_now.sort.each do |pkg_name|
+                # Already handled in this part of the layout
+                next if all_enabled_packages.include?(pkg_name)
+
+                # Not handled already. Import.
+                Autobuild::Package[pkg_name].import
+                manifest.load_package_manifest(pkg_name)
+                all_enabled_packages << pkg_name
+
+                # Add its dependencies to the next import set
+                Autobuild::Package[pkg_name].dependencies.each do |dep_name|
+                    if !Autobuild::Package[dep_name]
+                        raise ConfigError, "#{pkg_name} depends on #{dep_name}, but it does not seem to exist"
+                    elsif all_enabled_packages.include?(dep_name)
+                        next
+                    end
+                    packages_to_import << dep_name
+                end
             end
+            all_packages.merge(packages)
         end
+    end
 
-        libdir = File.join(prefix, "lib")
-        if File.directory?(libdir)
-            Autoproj.validate_solib_dependencies(libdir)
+    old_update_flag = Autobuild.do_update
+    begin
+        Autobuild.do_update = false
+        prepare_targets = all_packages.
+            find_all { |pkg_name| File.directory?(Autobuild::Package[pkg_name].srcdir) }.
+            map { |pkg_name| "#{pkg_name}-prepare" }
+        task "autoproj-prepare" => prepare_targets
+        Rake::Task["autoproj-prepare"].invoke
+    ensure
+        Autobuild.do_update = old_update_flag
+    end
+
+    if (mode =~ /build/)
+        missing_packages = all_packages.find_all { |pkg_name| !File.directory?(Autobuild::Package[pkg_name].srcdir) }
+        if missing_packages.empty?
+            # Backward compatibility: check if name/env.sh exists, and if it is
+            # the case, simply replace it with a link to the root one. And issue
+            # a warning
+            all_sublayouts.each do |name|
+                env_file = File.join(Autoproj.root_dir, name, "env.sh")
+                if name != '/' && File.file?(env_file)
+                    File.open(env_file, 'w') do |io|
+                        io.puts "source #{Autoproj.root_dir}/env.sh"
+                    end
+                end
+            end
+            Autoproj.export_env_sh
+        else
+            STDERR.puts color("WARN: #{missing_packages.join(", ")} are not yet imported, #{Autoproj.root_dir}/env.sh might not be up to date", :magenta)
         end
     end
 
-    if !all_env_sh.empty?
-        STDERR.puts <<-EOTEXT
-
-
-add the following lines at the bottom of your .bashrc:
-  #{all_env_sh.map { |name| "source #{Dir.pwd}#{name}env.sh" }.join("\n  ")}
-
-WARNING: autoproj will not work until your restart all
-your consoles, or run the following in them:
-  #{all_env_sh.map { |name| "$ source #{Dir.pwd}#{name}env.sh" }.join("\n  ")}
+    if !no_os_deps
+        if !all_enabled_packages.empty?
+            STDERR.puts color("autoproj: installing prepackaged dependencies", :bold)
+            manifest.install_os_dependencies(all_enabled_packages)
+        end
+    end
 
-        EOTEXT
+    if all_enabled_packages.empty?
+        STDERR.puts color("autoproj: nothing to do", :bold)
+    elsif Autobuild.do_build
+        if Autobuild.only_doc
+            STDERR.puts color("autoproj: building and installing documentation", :bold)
+        else
+            STDERR.puts color("autoproj: building and installing packages", :bold)
+        end
+        Autobuild.apply(all_enabled_packages, "autoproj-build")
     end
 
-rescue ConfigError => e
-    STDERR.puts
-    STDERR.puts color(e.message, :red, :bold)
-    if debug then raise
-    else exit 1
+    prefixes = all_enabled_packages.inject(Set.new) do |set, pkg_name|
+        set << Autobuild::Package[pkg_name].prefix
     end
-rescue Interrupt
-    STDERR.puts
-    STDERR.puts color("Interrupted by user", :red, :bold)
-    if debug then raise
-    else exit 1
+    prefixes.each do |prefix|
+        libdir = File.join(prefix, "lib")
+        if File.directory?(libdir)
+            Autoproj.validate_solib_dependencies(libdir)
+        end
     end
 end