autoproj 1.7.0.b3 → 1.7.0.rc1

data/History.txt

@@ -17,7 +17,12 @@
    in a separate non-trivial subfolder, and install it in a separate folder as
    well. This is meant to test the definition and handling of dependencies
    between the different packages.
- * added the autolocate tool, which returns the full path to a certain package
+ * added the autolocate tool, which returns the full path to a certain package.
+   The shell function "acd", akin to rosbuild's roscd, it provided in shell/
+ * added an easier way to add a local checked-out package to a build without
+   going through editing 20 files. Packages are automatically detected and added
+   as soon as they are explicitely added to autoproj/manifest.
+ * added the --stats option that displays timing statistics about the build
    
 = Version 1.6.2
  * overhauled osdeps fine-grained configuration. See

data/bin/autoproj

@@ -18,7 +18,7 @@ def color(*args)
     Autoproj.console.color(*args)
 end
 
-def report(debug)
+def report
     Autobuild::Reporting.report do
         yield
     end
@@ -34,13 +34,13 @@ rescue ConfigError => e
                 STDERR.puts color("  in #{path}", :red, :bold)
             end
     end
-    if debug then raise
+    if Autobuild.debug then raise
     else exit 1
     end
 rescue Interrupt
     STDERR.puts
     STDERR.puts color("Interrupted by user", :red, :bold)
-    if debug then raise
+    if Autobuild.debug then raise
     else exit 1
     end
 end
@@ -48,7 +48,7 @@ end
 Autoproj::OSDependencies.autodetect_ruby
 
 # Find the autoproj root dir
-report(Autobuild.debug) do
+report do
     selected_packages =
         begin Autoproj::CmdLine.parse_arguments(ARGV.dup)
         rescue RuntimeError => e
@@ -103,8 +103,14 @@ EOTEXT
         Autobuild.do_update = Autoproj.auto_update?
     end
 
-    selected_packages = Autoproj::CmdLine.resolve_user_selection(selected_packages)
-    Autoproj.manifest.explicit_selection = selected_packages
+    resolved_selected_packages = Autoproj::CmdLine.resolve_user_selection(selected_packages)
+    if !selected_packages.empty?
+        command_line_selection = resolved_selected_packages.dup
+    else
+        command_line_selection = Array.new
+    end
+    Autoproj.manifest.explicit_selection = resolved_selected_packages
+    selected_packages = resolved_selected_packages
     Autoproj::CmdLine.initial_package_setup
 
     # If in verbose mode, or if we only update sources, list the sources
@@ -192,9 +198,9 @@ EOTEXT
     if all_enabled_packages.empty?
         STDERR.puts color("autoproj: nothing to do", :bold)
     elsif Autoproj::CmdLine.doc?
-        Autoproj::CmdLine.build_packages(selected_packages, all_enabled_packages)
+        Autoproj::CmdLine.build_packages(command_line_selection, all_enabled_packages)
     elsif Autoproj::CmdLine.build? 
-        Autoproj::CmdLine.build_packages(selected_packages, all_enabled_packages)
+        Autoproj::CmdLine.build_packages(command_line_selection, all_enabled_packages)
 
         # Now, do some sanity checks on the result
         prefixes = all_enabled_packages.inject(Set.new) do |set, pkg_name|
@@ -213,7 +219,52 @@ EOTEXT
 
     if Autoproj::CmdLine.update_envsh?
         Autoproj.export_env_sh
-        STDERR.puts color("autoproj: updated #{Autoproj.root_dir}/env.sh", :green)
+        Autoproj.progress "autoproj: updated #{Autoproj.root_dir}/env.sh", :green
+    end
+
+    if Autoproj::CmdLine.show_statistics?
+        per_phase = Hash.new
+        per_type  = Hash.new
+
+        Autoproj.progress
+        Autoproj.progress "statistics about the build", :bold
+
+        Autoproj.manifest.each_package.sort_by(&:name).each do |pkg|
+            next if pkg.statistics.empty?
+
+            if per_phase.empty?
+                Autoproj.progress "detailed per package"
+            end
+
+            puts "  #{pkg.name}: %.1fs" % [pkg.statistics.values.inject(&:+)]
+            pkg.statistics.each_key.sort.each do |phase|
+                per_phase[phase] ||= 0
+                per_phase[phase] += pkg.statistics[phase]
+                per_type[pkg.class] ||= Hash.new
+                per_type[pkg.class][phase] ||= 0
+                per_type[pkg.class][phase] += pkg.statistics[phase]
+                puts "    #{phase}: %.1fs" % [pkg.statistics[phase]]
+            end
+        end
+
+        if !per_type.empty?
+            Autoproj.progress
+            Autoproj.progress "detailed per package type"
+            per_type.each do |pkg_type, phases|
+                Autoproj.progress "  #{pkg_type.name}: %.1fs" % [phases.values.inject(&:+)]
+                phases.each_key.sort.each do |phase_name|
+                    Autoproj.progress "    #{phase_name}: %.1fs" % [phases[phase_name]]
+                end
+            end
+        end
+
+        if !per_phase.empty?
+            Autoproj.progress
+            Autoproj.progress "summary per phase"
+            per_phase.keys.sort.each do |phase_name|
+                Autoproj.progress "  #{phase_name}: %.1fs" % [per_phase[phase_name]]
+            end
+        end
     end
 end
 

data/lib/autoproj/autobuild.rb

@@ -163,21 +163,38 @@ module Autoproj
         end
     end
 
+    def self.in_package_set(source, path)
+        @file_stack.push([source, File.expand_path(path).gsub(/^#{Regexp.quote(Autoproj.root_dir)}\//, '')])
+        yield
+    ensure
+        @file_stack.pop
+    end
+
     def self.import_autobuild_file(source, path)
         return if @loaded_autobuild_files.include?(path)
 
-        @file_stack.push([source, File.expand_path(path).gsub(/^#{Regexp.quote(Autoproj.root_dir)}\//, '')])
-        begin
-            Kernel.load path
-        rescue ConfigError => e
-            raise
-        rescue Exception => e
-            filter_load_exception(e, source, path)
+        in_package_set(source, File.expand_path(path).gsub(/^#{Regexp.quote(Autoproj.root_dir)}\//, '')) do
+            begin
+                Kernel.load path
+            rescue ConfigError => e
+                raise
+            rescue Exception => e
+                filter_load_exception(e, source, path)
+            end
+            @loaded_autobuild_files << path
         end
-        @loaded_autobuild_files << path
+    end
 
-    ensure
-        @file_stack.pop
+    # Tries to find a handler automatically for 'full_path'
+    def self.package_handler_for(full_path)
+        if !Dir.enum_for(:glob, File.join(full_path, "*.orogen")).to_a.empty?
+            "orogen_package"
+        elsif File.file?(File.join(full_path, "CMakeLists.txt"))
+            "cmake_package"
+        elsif File.directory?(File.join(full_path, "lib")) &&
+            !Dir.enum_for(:glob, File.join(full_path, "*.rb")).to_a.empty?
+            "ruby_package"
+        end
     end
 end
 

data/lib/autoproj/cmdline.rb

@@ -166,6 +166,29 @@ module Autoproj
             # source.yml files)
             manifest.load_importers
 
+            # Auto-add packages that are
+            #  * present on disk
+            #  * listed in the layout part of the manifest
+            #  * but have no definition
+            explicit = manifest.normalized_layout
+            explicit.each do |pkg_or_set, layout_level|
+                next if Autobuild::Package[pkg_or_set]
+                next if manifest.has_package_set?(pkg_or_set)
+
+                # This is not known. Check if we can auto-add it
+                full_path = File.expand_path(File.join(Autoproj.root_dir, layout_level, pkg_or_set))
+                next if !File.directory?(full_path)
+
+                if handler = Autoproj.package_handler_for(full_path)
+                    Autoproj.progress "  auto-adding #{pkg_or_set} #{"in #{layout_level} " if layout_level != "/"}using the #{handler.gsub(/_package/, '')} package handler"
+                    Autoproj.in_package_set(manifest.local_package_set, manifest.file) do
+                        send(handler, pkg_or_set)
+                    end
+                else
+                    Autoproj.warn "cannot auto-add #{pkg_or_set}: unknown package type"
+                end
+            end
+
             # We finished loading the configuration files. Not all configuration
             # is done (since we need to process the package setup blocks), but
             # save the current state of the configuration anyway.
@@ -335,6 +358,8 @@ module Autoproj
                 Autoproj.progress "  defines: #{set_packages.map(&:name).join(", ")}"
             end
 
+            packages_not_present = []
+
             Autoproj.progress
             Autoproj.progress("autoproj: packages", :bold)
             all_packages.to_a.sort_by(&:first).map(&:last).each do |pkg, pkg_set|
@@ -355,6 +380,7 @@ module Autoproj
                 Autoproj.progress "   #{vcs_def.to_s}"
 
                 if !File.directory?(pkg.srcdir)
+                    packages_not_present << pkg.name
                     Autoproj.progress "   NOT checked out yet, reported dependencies will be partial"
                 end
 
@@ -373,6 +399,15 @@ module Autoproj
                     Autoproj.progress "   disabled opt deps: #{opt_deps.join(", ")}"
                 end
             end
+
+            if !packages_not_present.empty?
+                Autoproj.progress
+                Autoproj.warn "the following packages are not yet checked out:"
+                packages_not_present.each_slice(4) do |*packages|
+                    Autoproj.warn "  #{packages.join(", ")}"
+                end
+                Autoproj.warn "therefore, the package list above and the listed dependencies are probably not complete"
+            end
         end
 
         # Returns the set of packages that are actually selected based on what
@@ -607,6 +642,8 @@ module Autoproj
         def self.doc?; @mode == "doc" end
         def self.snapshot?; @mode == "snapshot" end
 
+        def self.show_statistics?; !!@show_statistics end
+
         def self.osdeps?; @mode == "osdeps" end
         def self.show_osdeps?; @mode == "osdeps" && @show_osdeps end
         def self.revshow_osdeps?; @mode == "osdeps" && @revshow_osdeps end
@@ -723,6 +760,9 @@ where 'mode' is one of:
                     end
                     exit 0
                 end
+                opts.on('--stats', 'displays statistics about each of the phases in the package building process') do
+                    @show_statistics = true
+                end
 
                 opts.on("--with-depends", "apply rebuild and force-build to both packages selected on the command line and their dependencies") do
                     force_re_build_with_depends = true
@@ -936,7 +976,9 @@ where 'mode' is one of:
                     else pkg.autoproj_name
                     end
 
-                if !pkg.importer.respond_to?(:status)
+                if !pkg.importer
+                    lines << Autoproj.color("  is a local-only package (no VCS)", :bold, :red)
+                elsif !pkg.importer.respond_to?(:status)
                     lines << Autoproj.color("  the #{pkg.importer.class.name.gsub(/.*::/, '')} importer does not support status display", :bold, :red)
                 elsif !File.directory?(pkg.srcdir)
                     lines << Autoproj.color("  is not imported yet", :magenta)

data/lib/autoproj/manifest.rb

@@ -777,6 +777,8 @@ module Autoproj
             @disabled_imports = Set.new
             @moved_packages = Hash.new
 
+            @explicit_selection = Set.new
+
             @constant_definitions = Hash.new
             if Autoproj.has_config_key?('manifest_source')
                 @vcs = Autoproj.normalize_vcs_definition(Autoproj.user_config('manifest_source'))
@@ -995,6 +997,10 @@ module Autoproj
             each_package_set(*args, &block)
         end
 
+        def local_package_set
+            each_package_set.find { |s| s.kind_of?(LocalPackageSet) }
+        end
+
         # Save the currently known package sets. After this call,
         # #each_package_set will always return the same set regardless of
         # changes on the manifest's data structures
@@ -1224,6 +1230,12 @@ module Autoproj
             end
         end
 
+        # Returns true if +name+ is the name of a package set known to this
+        # autoproj installation
+        def has_package_set?(name)
+            each_package_set(false).find { |set| set.name == name }
+        end
+
         # +name+ can either be the name of a source or the name of a package. In
         # the first case, we return all packages defined by that source. In the
         # latter case, we return the singleton array [name]
@@ -1246,28 +1258,17 @@ module Autoproj
         #
         # If recursive is false, yields only the packages at this level.
         # Otherwise, return all packages.
-        def layout_packages(result, layout_def, recursive, validate = true)
-            layout_def.each do |value|
-                if !value.kind_of?(Hash) # sublayout
-                    begin
-                        pkgs = resolve_package_set(value)
-                        pkgs.each do |p|
-                            result << p
-                            Autobuild::Package[p].all_dependencies(result)
-                        end
-
-                    rescue ConfigError
-                        raise if validate
+        def layout_packages(result, validate)
+            normalized_layout.each_key do |pkg_or_set|
+                begin
+                    resolve_package_set(pkg_or_set).each do |pkg_name|
+                        result << pkg_name
+                        Autobuild::Package[pkg_name].all_dependencies(result)
                     end
+                rescue ConfigError
+                    raise if validate
                 end
             end
-
-            if recursive
-                each_sublayout(layout_def) do |sublayout_name, sublayout_def|
-                    layout_packages(result, sublayout_def, true)
-                end
-            end
-
             result
         end
 
@@ -1331,11 +1332,20 @@ module Autoproj
             end
         end
 
+        # Returns the set of packages that are selected by the layout
+        def all_selected_packages
+            result = default_packages.to_set
+            result.each do |pkg_name|
+                Autobuild::Package[pkg_name].all_dependencies(result)
+            end
+            result
+        end
+
         # Returns the set of packages that should be built if the user does not
         # specify any on the command line
         def default_packages(validate = true)
             names = if layout = data['layout']
-                        layout_packages(Set.new, layout, true, validate)
+                        layout_packages(Set.new, validate)
                     else
                         # No layout, all packages are selected
                         all_packages
@@ -1345,27 +1355,24 @@ module Autoproj
             names.to_set
         end
 
-        def search_layout(layout_level, layout_data, *names)
+        def normalized_layout(result = Hash.new { '/' }, layout_level = '/', layout_data = (data['layout'] || Hash.new))
             layout_data.each do |value|
                 if value.kind_of?(Hash)
-                    each_sublayout(value) do |subname, subdef|
-                        if result = search_layout("#{layout_level}/#{subname}", subdef)
-                            return result
-                        end
-                    end
-
-                elsif names.include?(value)
-                    return layout_level
+                    subname, subdef = value.find { true }
+                    normalized_layout(result, "#{layout_level}#{subname}/", subdef)
+                else
+                    result[value] = layout_level
                 end
             end
-            nil
+            result
         end
 
         # Returns the package directory for the given package name
         def whereis(package_name)
             Autoproj.in_file(self.file) do
                 set_name = definition_source(package_name).name
-                return (search_layout("/", (data['layout'] || Hash.new), package_name, set_name) || "/")
+                actual_layout = normalized_layout
+                return actual_layout[package_name] || actual_layout[set_name]
             end
         end
 
@@ -1464,17 +1471,17 @@ module Autoproj
         #  * as a package name
         #
         # This method converts the first two directories into the third one
-        def expand_package_selection(selected_packages)
+        def expand_package_selection(selection)
             base_dir = Autoproj.root_dir
 
             # The expanded selection
             expanded_packages = Set.new
             # All the packages that are available on this installation
-            all_layout_packages = self.all_layout_packages
+            all_layout_packages = self.all_selected_packages
 
             # First, remove packages that are directly referenced by name or by
             # package set names
-            selected_packages.each do |sel|
+            selection.each do |sel|
                 sel = Regexp.new(Regexp.quote(sel))
 
                 packages = all_layout_packages.
@@ -1493,7 +1500,7 @@ module Autoproj
 
             # Finally, check for package source directories
             all_packages = self.all_package_names
-            selected_packages.each do |sel|
+            selection.each do |sel|
                 match_pkg_name = Regexp.new(Regexp.quote(sel))
                 all_packages.each do |pkg_name|
                     pkg = Autobuild::Package[pkg_name]

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "1.7.0.b3"
+    VERSION = "1.7.0.rc1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: autoproj
 version: !ruby/object:Gem::Version 
-  hash: 6629749
+  hash: 977940582
   prerelease: true
   segments: 
   - 1
   - 7
   - 0
-  - b3
-  version: 1.7.0.b3
+  - rc1
+  version: 1.7.0.rc1
 platform: ruby
 authors: 
 - Sylvain Joyeux
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-24 00:00:00 +01:00
+date: 2010-11-26 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency