autoproj 1.5.2 → 1.5.3

data/History.txt

@@ -1,3 +1,26 @@
+= Version 1.5.3
+ * use osdeps-provided packages instead of autobuild-built ones even if they are
+   implicitely selected in the layout (i.e. because of a package set name).
+ * allow to force using the autobuild-built package instead of the OSdeps
+   provided one by explicitely listing the package name in the layout.
+ * fix multiple loading of manifests (harmless, but annoying)
+ * added an 'overrides' section in source.yml, and made the version_control
+   field local to the given package set. This is because there is too much
+   possibility for name clashes when people start sharing package sets, and it
+   allows to simply change some of the VCS parameters (most often, the branch)
+   without having to redefine the complete configuration.
+ * packages defined in the local source do not need to have an entry in
+   overrides.yml anymore. Only proper package sets do.
+ * fix env.sh update in cases of build failures
+ * fix osdeps/autobuild package unification in some cases. In particular,
+   autoproj will fall-back to self-built packages if an OS package is not
+   provided on *this particular OS/version pair*, even though it exists for
+   other OSes or other versions of the same OS.
+ * in case the layout selects some packages, partial build commands should not
+   forcefully select packages that are not part of the layout at all.
+ * add the ability to override the default.osdeps file by setting
+   AUTOPROJ_DEFAULT_OSDEPS environment variable
+
 = Version 1.5.2
  * fix ignored_packages and exclude_packages
 

data/bin/autoproj

@@ -9,6 +9,7 @@ require 'autoproj'
 require 'autoproj/autobuild'
 require 'open-uri'
 require 'autoproj/cmdline'
+require 'pp'
 
 require 'highline'
 include Autoproj
@@ -95,10 +96,10 @@ report(Autobuild.debug) do
     end
 
     selected_packages = Autoproj::CmdLine.resolve_user_selection(selected_packages)
+    Autoproj.manifest.explicit_selection = selected_packages
     Autoproj::CmdLine.initial_package_setup
 
     if Autoproj::CmdLine.only_status?
-        STDERR.puts
         all_enabled_packages = Autoproj::CmdLine.import_packages(selected_packages)
         Autoproj::CmdLine.status(all_enabled_packages)
         exit(0)
@@ -126,34 +127,36 @@ report(Autobuild.debug) do
         osdeps.install(Autoproj.build_system_dependencies)
     end
 
-    # Now, we actually do the import. Unfortunately, at this stage, we need to
-    # import the package and its dependencies ourselves so that we are able to
-    # build the list of packages that have actually been selected on the command
-    # line.
-    all_enabled_packages = Autoproj::CmdLine.import_packages(selected_packages)
-
-    if Autoproj::CmdLine.build? || Autoproj::CmdLine.doc? || Autoproj::CmdLine.update_os_dependencies?
-        if all_enabled_packages.empty?
-            STDERR.puts color("autoproj: nothing to do", :bold)
-        elsif Autoproj::CmdLine.build? || Autoproj::CmdLine.doc?
-            Autoproj::CmdLine.build_packages(selected_packages, all_enabled_packages)
-        elsif Autoproj::CmdLine.update_os_dependencies?
-            manifest.install_os_dependencies(all_enabled_packages)
-        end
+    begin
+        # Now, we actually do the import. Unfortunately, at this stage, we need to
+        # import the package and its dependencies ourselves so that we are able to
+        # build the list of packages that have actually been selected on the command
+        # line.
+        all_enabled_packages = Autoproj::CmdLine.import_packages(selected_packages)
 
-        # Now, do some sanity checks on the result
-        prefixes = all_enabled_packages.inject(Set.new) do |set, pkg_name|
-            set << Autobuild::Package[pkg_name].prefix
-        end
-        prefixes.each do |prefix|
-            libdir = File.join(prefix, "lib")
-            if File.directory?(libdir)
-                Autoproj.validate_solib_dependencies(libdir)
+        if Autoproj::CmdLine.build? || Autoproj::CmdLine.doc? || Autoproj::CmdLine.update_os_dependencies?
+            if all_enabled_packages.empty?
+                STDERR.puts color("autoproj: nothing to do", :bold)
+            elsif Autoproj::CmdLine.build? || Autoproj::CmdLine.doc?
+                Autoproj::CmdLine.build_packages(selected_packages, all_enabled_packages)
+            elsif Autoproj::CmdLine.update_os_dependencies?
+                manifest.install_os_dependencies(all_enabled_packages)
+            end
+
+            # Now, do some sanity checks on the result
+            prefixes = all_enabled_packages.inject(Set.new) do |set, pkg_name|
+                set << Autobuild::Package[pkg_name].prefix
+            end
+            prefixes.each do |prefix|
+                libdir = File.join(prefix, "lib")
+                if File.directory?(libdir)
+                    Autoproj.validate_solib_dependencies(libdir)
+                end
             end
         end
+    ensure
+        Autoproj.export_env_sh
+        STDERR.puts color("autoproj: updated #{Autoproj.root_dir}/env.sh", :green)
     end
-
-    Autoproj.export_env_sh
-    STDERR.puts color("autoproj: updated #{Autoproj.root_dir}/env.sh", :green)
 end
 

data/doc/guide/src/package_sets/importers.page

@@ -18,11 +18,15 @@ version_control:
 Autoproj follows the following rules to find the importer definition for a given
 package:
  
- - it looks *in reverse order* in the package sets listed in the manifest file,
-   and will take the definition from the *first* package set that has one.
- - in the source.yml file, it will match the package's name with the
-   package\_name fields. It will consider every matching block (i.e. every
-   package\_name that matches), overriding earlier options by later ones.
+ - it loads the information contained in the version_control section of the
+   package set that defines the considered package
+ - it will apply any modification that is contained in the overrides section by
+   the package sets listed *after* the aforementionned package set
+
+For both the version_control and overrides section, autoproj will match the
+package's name with the package\_name fields. It will consider every matching
+block (i.e. every package\_name that matches), overriding earlier setup by later
+ones.
 
 As an example, let's consider the following manifest:
 
@@ -47,10 +51,8 @@ version_control:
 Finally, the rubim.drivers package set has:
 
 {coderay:: ruby}
-version_control:
+overrides:
     - orocos/logger:
-        type: git
-        url: git://github.com/$PACKAGE.git
         branch: perf
 {coderay}
 
@@ -61,8 +63,8 @@ Then, the following will happen:
  * in addition, the 'branch' flag of orocos/orocos.rb will be overriden from
    'master' (the default) to 'roby'
  * in the case of orocos/logger, since there is a matching definition in
-   rubim.drivers, the definition in rubim.orocos *is not used at all*. Only the
-   definition found in rubim.drivers' source.yml file matters.
+   the overrides section of rubim.drivers, the 'perf' branch will be checked out
+   instead of the 'roby' branch, keeping the same importer type and URL
 
 The remaining of this page will detail the various options that exist to import
 a source package.

data/lib/autoproj/autobuild.rb

@@ -11,7 +11,7 @@ module Autobuild
 
         alias __depends_on__ depends_on
         def depends_on(name)
-            if Autoproj.osdeps.has?(name)
+            if Autoproj.osdeps.has?(name) && !Autoproj.manifest.explicitly_selected_package?(name)
                 @os_packages ||= Set.new
                 @os_packages << name
             else
@@ -328,12 +328,7 @@ class Autobuild::Git
     def snapshot(package, target_dir)
         Dir.chdir(package.srcdir) do
             head_commit   = `git rev-parse #{branch}`.chomp
-            {
-                'type' => 'git',
-                'url' => repository,
-                'commit' => head_commit,
-                'patches' => patches
-            }
+            { 'commit' => head_commit }
         end
     end
 end
@@ -344,16 +339,7 @@ class Autobuild::ArchiveImporter
         FileUtils.mkdir_p archive_dir
         FileUtils.cp @cachefile, archive_dir
 
-        {
-            'type' => 'archive',
-            'url' =>  File.join('$AUTOPROJ_SOURCE_DIR', File.basename(@cachefile)),
-            'mode' => @mode,
-            'update_cached_file' => false,
-            'patches' => patches,
-            'no_subdirectory' => @options[:no_subdirectory],
-            'archive_dir' => archive_dir
-
-        }
+        { 'url' =>  File.join('$AUTOPROJ_SOURCE_DIR', File.basename(@cachefile)) }
     end
 end
 

data/lib/autoproj/cmdline.rb

@@ -187,10 +187,10 @@ module Autoproj
                     source_yml = source.raw_description_file
                     STDERR.puts "  #{source_yml['name']}"
                     if source.local?
-                        STDERR.puts "    local source in #{source.local_dir}"
+                        puts "    local source in #{source.local_dir}"
                     else
-                        STDERR.puts "    from:  #{source.vcs}"
-                        STDERR.puts "    local: #{source.local_dir}"
+                        puts "    from:  #{source.vcs}"
+                        puts "    local: #{source.local_dir}"
                     end
 
                     lines = []
@@ -212,15 +212,17 @@ module Autoproj
                         w_col1 = col1.size if !w_col1 || col1.size > w_col1
                         w_col2 = col2.size if !w_col2 || col2.size > w_col2
                     end
-                    STDERR.puts "    packages:"
+                    puts "    packages:"
                     format = "    | %-#{w_col1}s | %-#{w_col2}s |"
                     lines.each do |col1, col2|
-                        STDERR.puts(format % [col1, col2])
+                        puts(format % [col1, col2])
                     end
                 end
             end
         end
 
+        # Returns the set of packages that are actually selected based on what
+        # the user gave on the command line
         def self.resolve_user_selection(selected_packages)
             manifest = Autoproj.manifest
 
@@ -280,7 +282,7 @@ module Autoproj
 
                 current_packages.
                     delete_if { |pkg| all_enabled_packages.include?(pkg.name) }
-                all_enabled_packages |= current_packages
+                all_enabled_packages |= current_packages.map(&:name).to_set
 
                 # We import first so that all packages can export the
                 # additional targets they provide.
@@ -319,7 +321,7 @@ module Autoproj
                 STDERR.puts "autoproj: finished importing packages"
             end
 
-            return all_enabled_packages.map(&:name)
+            return all_enabled_packages
         end
 
         def self.build_packages(selected_packages, all_enabled_packages)
@@ -950,10 +952,10 @@ EOTEXT
                 else Hash.new
                 end
 
-            if overrides['version_control']
-                overrides['version_control'].concat(version_control)
+            if overrides['overrides']
+                overrides['overrides'].concat(version_control)
             else
-                overrides['version_control'] = version_control
+                overrides['overrides'] = version_control
             end
 
             File.open(overrides_path, 'w') do |io|

data/lib/autoproj/default.osdeps

@@ -10,6 +10,8 @@ ruby18:
         - libopenssl-ruby1.8
     gentoo:
         - dev-lang/ruby:1.8
+    gem:
+        - rake
 
 ruby19:
     debian:
@@ -32,6 +34,8 @@ ruby19:
         - dev-lang/ruby:1.9
     arch:
         - ruby
+    gem:
+        - rake
 
 rdoc: gem
 

data/lib/autoproj/manifest.rb

@@ -198,7 +198,7 @@ module Autoproj
         def local?; vcs.local? end
         # True if this source defines nothing
         def empty?
-            !source_definition['version_control'] &&
+            !source_definition['version_control'] && !source_definition['overrides']
                 !each_package.find { true } &&
                 !File.exists?(File.join(raw_local_dir, "overrides.rb")) &&
                 !File.exists?(File.join(raw_local_dir, "init.rb"))
@@ -357,20 +357,25 @@ module Autoproj
             data
         end
 
+        # Returns the default importer for this package set
+        def default_importer
+            importer_definition_for('default')
+        end
+
         # Returns an importer definition for the given package, if one is
         # available. Otherwise returns nil.
         #
         # The returned value is a VCSDefinition object.
-        def importer_definition_for(package_name)
+        def version_control_field(package_name, section_name, validate = true)
             urls = source_definition['urls'] || Hash.new
             urls['HOME'] = ENV['HOME']
 
-            all_vcs     = source_definition['version_control']
+            all_vcs     = source_definition[section_name]
             if all_vcs
                 if all_vcs.kind_of?(Hash)
-                    raise ConfigError, "wrong format for the version_control field, you forgot the '-' in front of the package names"
+                    raise ConfigError, "wrong format for the #{section_name} section, you forgot the '-' in front of the package names"
                 elsif !all_vcs.kind_of?(Array)
-                    raise ConfigError, "wrong format for the version_control field"
+                    raise ConfigError, "wrong format for the #{section_name} section"
                 end
             end
 
@@ -435,14 +440,25 @@ module Autoproj
                 vcs_spec.dup.each do |name, value|
                     vcs_spec[name] = expand(value, expansions)
                 end
-                vcs_spec
 
-                Autoproj.normalize_vcs_definition(vcs_spec)
+                # If required, verify that the configuration is a valid VCS
+                # configuration
+                if validate
+                    Autoproj.normalize_vcs_definition(vcs_spec)
+                end
+                vcs_spec
             end
         rescue ConfigError => e
             raise ConfigError, "#{e.message} in #{source_file}", e.backtrace
         end
 
+        def importer_definition_for(package_name)
+            vcs_spec = version_control_field(package_name, 'version_control_field')
+            if vcs_spec
+                Autoproj.normalize_vcs_definition(vcs_spec)
+            end
+        end
+
         def each_package
             if !block_given?
                 return enum_for(:each_package)
@@ -471,6 +487,12 @@ module Autoproj
             File.join(Autoproj.config_dir, "overrides.yml")
         end
 
+        # Returns the default importer for this package set
+        def default_importer
+            importer_definition_for('default') ||
+                Autoproj.normalize_vcs_definition(:type => 'none')
+        end
+
         def raw_description_file
             path = source_file
             if File.file?(path)
@@ -512,6 +534,15 @@ module Autoproj
             end
         end
 
+        # The set of packages that are selected by the user, either through the
+        # manifest file or through the command line, as a set of package names
+        attr_accessor :explicit_selection
+
+        # Returns true if +pkg_name+ has been explicitely selected
+        def explicitly_selected_package?(pkg_name)
+            explicit_selection && explicit_selection.include?(pkg_name)
+        end
+
 	def self.load(file)
             begin
                 data = YAML.load(File.read(file))
@@ -666,6 +697,13 @@ module Autoproj
             end
 
             if @sources
+                if load_description
+                    @sources.each do |src|
+                        if !src.source_definition
+                            src.load_description_file
+                        end
+                    end
+                end
                 return @sources.each(&block)
             end
 
@@ -687,9 +725,7 @@ module Autoproj
             end
 
             all_sources.each(&block)
-            if load_description
-                @sources = all_sources
-            end
+            @sources = all_sources
         end
 
         # Register a new package
@@ -802,15 +838,26 @@ module Autoproj
                     package_set
             end
 
-            each_source.to_a.reverse.each do |source|
-                vcs = source.importer_definition_for(package_name)
-                if vcs
-                    return vcs
-                elsif package_source.name == source.name
-                    return
+            sources = each_source.to_a
+
+            # Remove sources listed before the package source
+            while !sources.empty? && sources[0].name != package_source.name
+                sources.shift
+            end
+            package_source = sources.shift
+
+            # Get the version control information from the package source. There
+            # must be one
+            vcs_spec = package_source.version_control_field(package_name, 'version_control')
+            return if !vcs_spec
+
+            sources.each do |src|
+                overrides_spec = src.version_control_field(package_name, 'overrides', false)
+                if overrides_spec
+                    vcs_spec.merge!(overrides_spec)
                 end
             end
-            nil
+            Autoproj.normalize_vcs_definition(vcs_spec)
         end
 
         # Sets up the package importers based on the information listed in
@@ -832,7 +879,7 @@ module Autoproj
         def load_importers
             packages.each_value do |pkg|
                 vcs = importer_definition_for(pkg.autobuild.name, pkg.package_set) ||
-                    importer_definition_for("default", pkg.package_set)
+                    pkg.package_set.default_importer
 
                 if vcs
                     Autoproj.add_build_system_dependency vcs.type
@@ -855,8 +902,9 @@ module Autoproj
                     raise ConfigError, "#{name} is neither a package nor a source"
                 end
                 packages.values.
-                    map { |pkg| pkg.autobuild.name if pkg.package_set.name == source.name }.
-                    compact
+                    find_all { |pkg| pkg.package_set.name == source.name }.
+                    map { |pkg| pkg.autobuild.name }.
+                    find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
             end
         end
 
@@ -921,29 +969,25 @@ module Autoproj
             end
         end
 
-        def in_sublayout(name, packages)
-            srcdir  = File.join(Autoproj.root_dir, name)
-            prefix  = File.join(Autoproj.build_dir, name)
-            logdir  = File.join(prefix, "log")
-            Autobuild.logdir = logdir
-            packages.each do |pkg_name|
-                pkg = Autobuild::Package[pkg_name]
-                pkg.srcdir = File.join(srcdir, pkg_name)
-                pkg.prefix = prefix
-                pkg.logdir = logdir
-            end
-
-            yield
+        # Returns the set of package names that are available for installation.
+        # It is the set of packages listed in the layout minus the exluded and
+        # ignred ones
+        def all_package_names
+            default_packages
         end
 
+        # Returns the set of packages that should be built if the user does not
+        # specify any on the command line
         def default_packages
             names = if layout = data['layout']
                         layout_packages(layout, true)
                     else
                         # No layout, all packages are selected
                         packages.values.
-                            map { |pkg| pkg.autobuild.name }
+                            map { |pkg| pkg.autobuild.name }.
+                            find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
                     end
+
             names.delete_if { |pkg_name| excluded?(pkg_name) || ignored?(pkg_name) }
             names.to_set
         end
@@ -1024,25 +1068,25 @@ module Autoproj
         def expand_package_selection(selected_packages)
             base_dir = Autoproj.root_dir
 
-            expanded_packages = []
-
-            # Get all the package names
-            package_names = Autobuild::Package.each(true).
-                map do |name, pkg|
-                    pkg.name
-                end
+            # The expanded selection
+            expanded_packages = Set.new
+            # All the packages that are available on this installation
+            all_package_names = self.all_package_names
 
             # First, remove packages that are directly referenced by name or by
             # package set names
             selected_packages.delete_if do |sel|
                 sel = Regexp.new(Regexp.quote(sel))
 
-                packages = package_names.find_all { |pkg_name| pkg_name =~ sel }
-                expanded_packages.concat(packages)
+                packages = all_package_names.
+                    find_all { |pkg_name| pkg_name =~ sel }.
+                    to_set
+                expanded_packages |= packages
 
                 sources = each_source.find_all { |source| source.name =~ sel }
                 sources.each do |source|
-                    expanded_packages.concat(resolve_package_set(source.name))
+                    packages = resolve_package_set(source.name).to_set
+                    expanded_packages |= (packages & all_package_names)
                 end
 
                 !packages.empty? && !sources.empty?
@@ -1056,12 +1100,13 @@ module Autoproj
             each_package_set(nil) do |layout_name, packages, _|
                 selected_packages.delete_if do |sel|
                     if layout_name[0..-1] =~ Regexp.new("#{sel}\/?$")
-                        expanded_packages.concat(packages.to_a)
+                        expanded_packages |= packages.to_set
                     else
                         match = Regexp.new("^#{Regexp.quote(sel)}")
-                        Autobuild::Package.each(true) do |name, pkg|
+                        all_package_names.each do |pkg_name|
+                            pkg = Autobuild::Package[pkg_name]
                             if pkg.srcdir =~ match
-                                expanded_packages << name
+                                expanded_packages << pkg_name
                             end
                         end
                     end

data/lib/autoproj/osdeps.rb

@@ -31,7 +31,16 @@ module Autoproj
 
         AUTOPROJ_OSDEPS = File.join(File.expand_path(File.dirname(__FILE__)), 'default.osdeps')
         def self.load_default
-            @default_osdeps ||= OSDependencies.load(AUTOPROJ_OSDEPS)
+            if @default_osdeps
+                @default_osdeps
+            else
+                file = ENV['AUTOPROJ_DEFAULT_OSDEPS'] || AUTOPROJ_OSDEPS
+                if !File.file?(file)
+                    STDERR.puts "WARN: #{file} (from AUTOPROJ_DEFAULT_OSDEPS) is not a file, falling back to #{AUTOPROJ_OSDEPS}"
+                    file = AUTOPROJ_OSDEPS
+                end
+                @default_osdeps = OSDependencies.load(file)
+            end
         end
 
         attr_reader :definitions
@@ -128,14 +137,18 @@ module Autoproj
             'arch' => 'pacman -Sy --noconfirm %s'
         }
 
-        def generate_os_script(dependencies)
+        # Resolves the given OS dependencies into the actual packages that need
+        # to be installed on this particular OS.
+        #
+        # Raises ConfigError if some packages can't be found
+        def resolve_os_dependencies(dependencies)
             os_name, os_version = operating_system
             if !OS_PACKAGE_INSTALL.has_key?(os_name)
                 raise ConfigError, "I don't know how to install packages on #{os_name}"
             end
 
-            shell_snippets = ""
             os_packages    = []
+            shell_snippets = []
             dependencies.each do |name|
                 dep_def = definitions[name]
                 if !dep_def
@@ -154,6 +167,7 @@ module Autoproj
                 end
 
                 data = os_entry.last
+
                 # This package does not need to be installed on this operating system (example: build tools on Gentoo)
                 next if !data || data == "ignore"
 
@@ -177,20 +191,29 @@ module Autoproj
                 elsif data.to_str =~ /\w+/
                     os_packages << data.to_str
                 else
-                    shell_snippets << "\n" << data << "\n"
+                    shell_snippets << data.to_str
                 end
             end
 
+            return os_packages, shell_snippets
+        end
+
+
+        def generate_os_script(dependencies)
+            os_name, os_version = operating_system
+            os_packages, shell_snippets = resolve_os_dependencies(dependencies)
+
             "#! /bin/bash\n" +
             GAIN_ROOT_ACCESS + "\n" +
                 (OS_PACKAGE_INSTALL[os_name] % [os_packages.join(" ")]) +
-                "\n" + shell_snippets
+                "\n" + shell_snippets.join("\n")
         end
 
         # Returns true if there is an operating-system package with that name,
         # and false otherwise
         def has?(name)
-            partition_packages([name].to_set)
+            osdeps, gemdeps = partition_packages([name].to_set)
+            resolve_os_dependencies(osdeps)
             true
         rescue ConfigError
             false

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "1.5.2"
+    VERSION = "1.5.3"
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 1
   - 5
-  - 2
-  version: 1.5.2
+  - 3
+  version: 1.5.3
 platform: ruby
 authors: 
 - Sylvain Joyeux
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-28 00:00:00 +02:00
+date: 2010-06-08 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -97,8 +97,8 @@ dependencies:
         segments: 
         - 2
         - 0
-        - 3
-        version: 2.0.3
+        - 4
+        version: 2.0.4
   type: :development
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
@@ -139,8 +139,8 @@ dependencies:
         segments: 
         - 2
         - 6
-        - 0
-        version: 2.6.0
+        - 1
+        version: 2.6.1
   type: :development
   version_requirements: *id009
 description: |-