autoproj 1.13.7 → 2.0.0.b1

data/lib/autoproj/metapackage.rb

@@ -38,14 +38,6 @@ module Autoproj
             @packages.each(&block)
         end
 
-        # Filter this metapackage by removing some packages
-        def remove(pkg)
-            if !pkg.respond_to?(:to_str)
-                pkg = pkg.name
-            end
-            @packages.delete_if { |p| p.name == pkg }
-        end
-
         # Tests if the given package is included in this metapackage
         #
         # @param [String,#name] pkg the package or package name

data/lib/autoproj/ops/build.rb

@@ -9,14 +9,9 @@ module Autoproj
             # The manifest on which we operate
             # @return [Manifest]
             attr_reader :manifest
-            # Whether we should update the OS dependencies before building. It
-            # controls the behaviour of {rebuild_all} as well as
-            # {build_packages}
-            attr_predicate :update_os_dependencies?, true
 
-            def initialize(manifest, update_os_dependencies = true)
+            def initialize(manifest)
                 @manifest = manifest
-                self.update_os_dependencies = update_os_dependencies
             end
 
             # Triggers a rebuild of all packages
@@ -26,17 +21,6 @@ module Autoproj
             # non-OS-specific managers that support it (e.g. RubyGems) if
             # {update_os_dependencies?} is set to true (the default)
             def rebuild_all
-                if update_os_dependencies?
-                    # We also reinstall the osdeps that provide the
-                    # functionality
-                    managers = Autoproj.osdeps.setup_package_handlers
-                    managers.each do |mng|
-                        if mng.respond_to?(:reinstall)
-                            mng.reinstall
-                        end
-                    end
-                end
-
                 packages = manifest.all_layout_packages
                 manifest.pristine_os_dependencies(packages)
                 rebuild_packages(packages, packages)

data/lib/autoproj/ops/configuration.rb

@@ -6,35 +6,19 @@ module Autoproj
 
     # Implementation of the operations to manage the configuration
     class Configuration
-        # The manifest object that represents the autoproj configuration
-        #
-        # @return [Manifest]
-        attr_reader :manifest
-
-        # The loader object that should be used to load files such as init.rb
-        attr_reader :loader
-
-        # The main configuration directory
-        #
-        # @return [String]
-        attr_reader :config_dir
+        attr_reader :ws
 
         # The autoproj install we should update from (if any)
         #
         # @return [nil,InstallationManifest]
         attr_reader :update_from
 
-        # The object that allows us to install OS dependencies
-        def osdeps
-            Autoproj.osdeps
-        end
-
         # The path in which remote package sets should be exposed to the
         # user
         #
         # @return [String]
         def remotes_dir
-            Autoproj.remotes_dir
+            ws.remotes_dir
         end
 
         # The path in which remote package sets should be exposed to the
@@ -42,45 +26,44 @@ module Autoproj
         #
         # @return [String]
         def remotes_user_dir
-            File.join(config_dir, "remotes")
+            File.join(ws.config_dir, "remotes")
         end
 
         # The path to the main manifest file
         #
         # @return [String]
         def manifest_path
-            File.join(config_dir, 'manifest')
+            File.join(ws.config_dir, 'manifest')
         end
 
-
         # @param [Manifest] manifest
         # @param [Loader] loader
         # @option options [InstallationManifest] :update_from
-        #   (CmdLine.update_from) another autoproj installation from which we
+        #   another autoproj installation from which we
         #   should update (instead of the normal VCS)
-        def initialize(manifest, loader, options = Hash.new)
-            options = Kernel.validate_options options,
-                :update_from => CmdLine.update_from
-            @manifest = manifest
-            @loader = loader
+        def initialize(workspace, options = Hash.new)
+            options = validate_options options,
+                update_from: nil
+            @ws = workspace
             @update_from = options[:update_from]
-            @config_dir = Autoproj.config_dir
             @remote_update_message_displayed = false
         end
 
         # Imports or updates a source (remote or otherwise).
         #
         # See create_autobuild_package for informations about the arguments.
-        def self.update_configuration_repository(vcs, name, into, options = Hash.new)
-            options = Kernel.validate_options options, update_from: nil, only_local: false
-            update_from, only_local = options.values_at(:update_from, :only_local)
+        def update_configuration_repository(vcs, name, into, options = Hash.new)
+            options = Kernel.validate_options options,
+                only_local: false,
+                checkout_only: !Autobuild.do_update,
+                ignore_errors: false
 
             fake_package = Tools.create_autobuild_package(vcs, name, into)
             if update_from
                 # Define a package in the installation manifest that points to
                 # the desired folder in other_root
                 relative_path = Pathname.new(into).
-                    relative_path_from(Pathname.new(Autoproj.root_dir)).to_s
+                    relative_path_from(Pathname.new(root_dir)).to_s
                 other_dir = File.join(update_from.path, relative_path)
                 if File.directory?(other_dir)
                     update_from.packages.unshift(
@@ -96,7 +79,7 @@ module Autoproj
                     fake_package.update = false
                 end
             end
-            fake_package.import(only_local: only_local)
+            fake_package.import(options)
 
         rescue Autobuild::ConfigException => e
             raise ConfigError.new, "cannot import #{name}: #{e.message}", e.backtrace
@@ -106,13 +89,20 @@ module Autoproj
         #
         # @return [Boolean] true if something got updated or checked out,
         #   and false otherwise
-        def update_main_configuration(only_local = false)
-            self.class.update_configuration_repository(
-                manifest.vcs,
+        def update_main_configuration(options = Hash.new)
+            if !options.kind_of?(Hash)
+                options = Hash[only_local: options]
+            end
+            options = validate_options options,
+                only_local: false,
+                checkout_only: !Autobuild.do_update,
+                ignore_errors: false
+
+            update_configuration_repository(
+                ws.manifest.vcs,
                 "autoproj main configuration",
-                config_dir,
-                update_from: update_from,
-                only_local: only_local)
+                ws.config_dir,
+                options)
         end
 
         # Update or checkout a remote package set, based on its VCS definition
@@ -120,11 +110,20 @@ module Autoproj
         # @param [VCSDefinition] vcs the package set VCS
         # @return [Boolean] true if something got updated or checked out,
         #   and false otherwise
-        def update_remote_package_set(vcs, only_local = false)
-            name = PackageSet.name_of(manifest, vcs)
+        def update_remote_package_set(vcs, options = Hash.new)
+            # BACKWARD
+            if !options.kind_of?(Hash)
+                options = Hash[only_local: options]
+            end
+            options = validate_options options,
+                only_local: false,
+                checkout_only: !Autobuild.do_update,
+                ignore_errors: false
+
+            name = PackageSet.name_of(ws.manifest, vcs)
             raw_local_dir = PackageSet.raw_local_dir_of(vcs)
 
-            return if !Autobuild.do_update && File.exists?(raw_local_dir)
+            return if options[:checkout_only] && File.exists?(raw_local_dir)
 
             # YUK. I am stopping there in the refactoring
             # TODO: figure out a better way
@@ -132,11 +131,9 @@ module Autoproj
                 Autoproj.message("autoproj: updating remote definitions of package sets", :bold)
                 @remote_update_message_displayed = true
             end
-            osdeps.install([vcs.type])
-            self.class.update_configuration_repository(
-                vcs, name, raw_local_dir,
-                update_from: update_from,
-                only_local: only_local)
+            ws.osdeps.install([vcs.type])
+            update_configuration_repository(
+                vcs, name, raw_local_dir, options)
         end
 
         # Create the user-visible directory for a remote package set
@@ -144,7 +141,7 @@ module Autoproj
         # @param [VCSDefinition] vcs the package set VCS
         # @return [String] the full path to the created user dir
         def create_remote_set_user_dir(vcs)
-            name = PackageSet.name_of(manifest, vcs)
+            name = PackageSet.name_of(ws.manifest, vcs)
             raw_local_dir = PackageSet.raw_local_dir_of(vcs)
             FileUtils.mkdir_p(remotes_user_dir)
             symlink_dest = File.join(remotes_user_dir, name)
@@ -166,9 +163,9 @@ module Autoproj
         end
 
         def load_package_set(vcs, options, imported_from)
-            pkg_set = PackageSet.new(manifest, vcs)
+            pkg_set = PackageSet.new(ws.manifest, vcs)
             pkg_set.auto_imports = options[:auto_imports]
-            loader.load_if_present(pkg_set, pkg_set.local_dir, 'init.rb')
+            ws.load_if_present(pkg_set, pkg_set.local_dir, 'init.rb')
             pkg_set.load_description_file
             if imported_from
                 pkg_set.imported_from << imported_from
@@ -204,16 +201,22 @@ module Autoproj
         #
         # @yieldparam [String] osdep the name of an osdep required to import the
         #   package sets
-        def load_and_update_package_sets(root_pkg_set, only_local = false)
+        def load_and_update_package_sets(root_pkg_set, options = Hash.new)
+            if !options.kind_of?(Hash)
+                options = Hash[only_local: options]
+            end
+            options = validate_options options,
+                only_local: false,
+                checkout_only: !Autobuild.do_update,
+                ignore_errors: false
+
             package_sets = [root_pkg_set]
             by_repository_id = Hash.new
             by_name = Hash.new
 
-            required_remotes_dirs = Array.new
-
             queue = queue_auto_imports_if_needed(Array.new, root_pkg_set, root_pkg_set)
             while !queue.empty?
-                vcs, options, imported_from = queue.shift
+                vcs, import_options, imported_from = queue.shift
                 repository_id = repository_id_of(vcs)
                 if already_processed = by_repository_id[repository_id]
                     already_processed_vcs, already_processed_from, pkg_set = *already_processed
@@ -229,28 +232,17 @@ module Autoproj
                 end
                 by_repository_id[repository_id] = [vcs, imported_from]
 
-                # Make sure the package set has been already checked out to
-                # retrieve the actual name of the package set
                 if !vcs.local?
-                    update_remote_package_set(vcs, only_local)
-                    raw_local_dir = PackageSet.raw_local_dir_of(vcs)
-                    required_remotes_dirs << raw_local_dir
+                    update_remote_package_set(vcs, options)
+                    create_remote_set_user_dir(vcs)
                 end
-                name = PackageSet.name_of(manifest, vcs)
-
-                required_user_dirs = by_name.collect { |k,v| k }
-                Autoproj.debug "Trying to load package_set: #{name} from definition #{repository_id}"
-                Autoproj.debug "Already loaded package_sets are: #{required_user_dirs}"
 
+                name = PackageSet.name_of(ws.manifest, vcs)
                 if already_loaded = by_name[name]
                     already_loaded_pkg_set, already_loaded_vcs = *already_loaded
                     if already_loaded_vcs != vcs
                         if imported_from
-                            Autoproj.warn "redundant auto-import by #{imported_from.name} for package set '#{name}'."
-                            Autoproj.warn "    A package set with the same name (#{name}) has already been imported from"
-                            Autoproj.warn "        #{already_loaded_vcs}"
-                            Autoproj.warn "    Skipping the following one: "
-                            Autoproj.warn "        #{vcs}"
+                            Autoproj.warn "#{imported_from.name} auto-imports a package set from #{vcs}, but a package set with the same name (#{name}) has already been imported from #{already_loaded_vcs}, I am skipping this one"
                         else
                             Autoproj.warn "the manifest refers to a package set from #{vcs}, but a package set with the same name (#{name}) has already been imported from #{already_loaded_vcs}, I am skipping this one"
                         end
@@ -261,11 +253,9 @@ module Autoproj
                         imported_from.imports << already_loaded_pkg_set
                     end
                     next
-                else
-                    create_remote_set_user_dir(vcs)
                 end
 
-                pkg_set = load_package_set(vcs, options, imported_from)
+                pkg_set = load_package_set(vcs, import_options, imported_from)
                 by_repository_id[repository_id][2] = pkg_set
                 package_sets << pkg_set
 
@@ -275,21 +265,18 @@ module Autoproj
                 queue_auto_imports_if_needed(queue, pkg_set, root_pkg_set)
             end
 
-            required_user_dirs = by_name.collect { |k,v| k }
-            cleanup_remotes_dir(package_sets, required_remotes_dirs)
-            cleanup_remotes_user_dir(package_sets, required_user_dirs)
+            cleanup_remotes_dir(package_sets)
+            cleanup_remotes_user_dir(package_sets)
             package_sets
         end
 
         # Removes from {remotes_dir} the directories that do not match a package
         # set
-        def cleanup_remotes_dir(package_sets = manifest.package_sets, required_remotes_dirs = Array.new)
+        def cleanup_remotes_dir(package_sets = ws.manifest.package_sets)
             # Cleanup the .remotes and remotes_symlinks_dir directories
             Dir.glob(File.join(remotes_dir, '*')).each do |dir|
                 dir = File.expand_path(dir)
-                # Once a package set has been checked out during the process,
-                # keep it -- so that it won't be checked out again
-                if File.directory?(dir) && !required_remotes_dirs.include?(dir)
+                if File.directory?(dir) && !package_sets.find { |pkg| pkg.raw_local_dir == dir }
                     FileUtils.rm_rf dir
                 end
             end
@@ -297,11 +284,10 @@ module Autoproj
 
         # Removes from {remotes_user_dir} the directories that do not match a
         # package set
-        def cleanup_remotes_user_dir(package_sets = manifest.package_sets, required_user_dirs = Array.new)
+        def cleanup_remotes_user_dir(package_sets = ws.manifest.package_sets)
             Dir.glob(File.join(remotes_user_dir, '*')).each do |file|
                 file = File.expand_path(file)
-                user_dir = File.basename(file)
-                if File.symlink?(file) && !required_user_dirs.include?(user_dir)
+                if File.symlink?(file) && !package_sets.find { |pkg_set| pkg_set.user_local_dir == file }
                     FileUtils.rm_f file
                 end
             end
@@ -349,26 +335,42 @@ module Autoproj
             sorted
         end
 
-        def update_configuration(only_local = false)
+        def load_package_sets(options = Hash.new)
+            options = validate_options options,
+                only_local: false,
+                checkout_only: true,
+                ignore_errors: false
+            update_configuration(options)
+        end
+
+        def update_configuration(options = Hash.new)
+            if !options.kind_of?(Hash)
+                options = Hash[only_local: options]
+            end
+            options = validate_options options,
+                only_local: false,
+                checkout_only: !Autobuild.do_update,
+                ignore_errors: false
+
             # Load the installation's manifest a first time, to check if we should
             # update it ... We assume that the OS dependencies for this VCS is already
             # installed (i.e. that the user did not remove it)
-            if manifest.vcs && !manifest.vcs.local?
-                update_main_configuration(only_local)
+            if ws.manifest.vcs && !ws.manifest.vcs.local?
+                update_main_configuration(options)
             end
-            Tools.load_main_initrb(manifest)
-            manifest.load(manifest_path)
+            ws.load_main_initrb
+            ws.manifest.load(manifest_path)
 
-            root_pkg_set = manifest.local_package_set
+            root_pkg_set = ws.manifest.local_package_set
             root_pkg_set.load_description_file
             root_pkg_set.explicit = true
-            package_sets = load_and_update_package_sets(root_pkg_set, only_local)
+            package_sets = load_and_update_package_sets(root_pkg_set, options)
             root_pkg_set.imports.each do |pkg_set|
                 pkg_set.explicit = true
             end
             package_sets = sort_package_sets_by_import_order(package_sets, root_pkg_set)
             package_sets.each do |pkg_set|
-                manifest.register_package_set(pkg_set)
+                ws.manifest.register_package_set(pkg_set)
             end
             # YUK. I am stopping there in the refactoring
             # TODO: figure out a better way

data/lib/autoproj/ops/import.rb

@@ -0,0 +1,222 @@
+module Autoproj
+    module Ops
+        class Import
+            attr_reader :ws
+            def initialize(ws)
+                @ws = ws
+            end
+
+            def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
+                root = !reason
+                chain.unshift pkg_name
+                if root
+                    reason = ws.manifest.exclusion_reason(pkg_name)
+                else
+                    if chain.size == 1
+                        ws.manifest.add_exclusion(pkg_name, "its dependency #{reason}")
+                    else
+                        ws.manifest.add_exclusion(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
+                    end
+                end
+
+                return if !revdeps.has_key?(pkg_name)
+                revdeps[pkg_name].each do |dep_name|
+                    if !ws.manifest.excluded?(dep_name)
+                        mark_exclusion_along_revdeps(dep_name, revdeps, chain.dup, reason)
+                    end
+                end
+            end
+
+            VALID_OSDEP_AVAILABILITY =
+                [OSDependencies::AVAILABLE, OSDependencies::IGNORE]
+
+            def import_next_step(pkg, reverse_dependencies)
+                new_packages = []
+                pkg.dependencies.each do |dep_name|
+                    reverse_dependencies[dep_name] << pkg.name
+                    new_packages << ws.manifest.find_autobuild_package(dep_name)
+                end
+                pkg_opt_deps, pkg_opt_os_deps = pkg.partition_optional_dependencies
+                pkg_opt_deps.each do |dep_name|
+                    new_packages << ws.manifest.find_autobuild_package(dep_name)
+                end
+
+                # Handle OS dependencies, excluding the package if some
+                # dependencies are not available
+                pkg.os_packages.each do |dep_name|
+                    reverse_dependencies[dep_name] << pkg.name
+                end
+                (pkg.os_packages | pkg_opt_os_deps).each do |dep_name|
+                    if ws.manifest.excluded?(dep_name)
+                        mark_exclusion_along_revdeps(dep_name, reverse_dependencies)
+                    end
+                end
+
+                new_packages.delete_if do |new_pkg|
+                    if ws.manifest.excluded?(new_pkg.name)
+                        mark_exclusion_along_revdeps(new_pkg.name, reverse_dependencies)
+                        true
+                    elsif ws.manifest.ignored?(new_pkg.name)
+                        true
+                    end
+                end
+                new_packages
+            end
+
+            def import_packages(selection, options = Hash.new)
+                options, import_options = Kernel.filter_options options,
+                    warn_about_ignored_packages: true,
+                    warn_about_excluded_packages: true,
+                    recursive: true
+
+                manifest = ws.manifest
+
+                updated_packages = Array.new
+                selected_packages = selection.each_source_package_name.map do |pkg_name|
+                    manifest.find_autobuild_package(pkg_name)
+                end.to_set
+
+                # The set of all packages that are currently selected by +selection+
+                all_processed_packages = Set.new
+                # The reverse dependencies for the package tree. It is discovered as
+                # we go on with the import
+                #
+                # It only contains strong dependencies. Optional dependencies are
+                # not included, as we will use this only to take into account
+                # package exclusion (and that does not affect optional dependencies)
+                reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }
+
+                package_queue = selected_packages.to_a.sort_by(&:name)
+                while !package_queue.empty?
+                    pkg = package_queue.shift
+                    # Remove packages that have already been processed
+                    next if all_processed_packages.include?(pkg.name)
+                    all_processed_packages << pkg.name
+
+                    # Try to auto-exclude the package early. If the autobuild file
+                    # contained some information that allows us to exclude it now,
+                    # then let's just do it
+                    import_next_step(pkg, reverse_dependencies)
+                    if manifest.excluded?(pkg.name)
+                        selection.filter_excluded_and_ignored_packages(manifest)
+                        next
+                    elsif manifest.ignored?(pkg.name)
+                        next
+                    end
+
+                    # If the package has no importer, the source directory must
+                    # be there
+                    if !pkg.importer && !File.directory?(pkg.srcdir)
+                        raise ConfigError.new, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
+                    end
+
+                    ## COMPLETELY BYPASS RAKE HERE
+                    # The reason is that the ordering of import/prepare between
+                    # packages is not important BUT the ordering of import vs.
+                    # prepare in one package IS important: prepare is the method
+                    # that takes into account dependencies.
+                    pkg.import(import_options)
+                    if pkg.updated?
+                        updated_packages << pkg.name
+                    end
+                    Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
+                    manifest.load_package_manifest(pkg.name)
+
+                    # The package setup mechanisms might have added an exclusion
+                    # on this package. Handle this.
+                    if manifest.excluded?(pkg.name)
+                        mark_exclusion_along_revdeps(pkg.name, reverse_dependencies)
+                        # Run a filter now, to have errors as early as possible
+                        selection.filter_excluded_and_ignored_packages(manifest)
+                        # Delete this package from the current_packages set
+                        next
+                    elsif manifest.ignored?(pkg.name)
+                        next
+                    end
+
+                    Autoproj.each_post_import_block(pkg) do |block|
+                        block.call(pkg)
+                    end
+                    pkg.update_environment
+
+                    new_packages = import_next_step(pkg, reverse_dependencies)
+
+                    # Excluded dependencies might have caused the package to be
+                    # excluded as well ... do not add any dependency to the
+                    # processing queue if it is the case
+                    if !manifest.excluded?(pkg.name) && options[:recursive]
+                        package_queue.concat(new_packages.sort_by(&:name))
+                    end
+
+                    # Verify that everything is still OK with the new
+                    # exclusions/ignores
+                    selection.filter_excluded_and_ignored_packages(manifest)
+                end
+
+                # Now run optional dependency resolution. This is done now, as
+                # some optional dependencies might have been excluded during the
+                # resolution process above
+                all_enabled_sources, all_enabled_osdeps =
+                    Set.new, selection.each_osdep_package_name.to_set
+                package_queue = selection.each_source_package_name.to_a
+                while !package_queue.empty?
+                    pkg_name = package_queue.shift
+                    next if all_enabled_sources.include?(pkg_name)
+                    all_enabled_sources << pkg_name
+
+                    pkg = manifest.find_autobuild_package(pkg_name)
+                    packages, osdeps = pkg.partition_optional_dependencies
+                    packages.each do |pkg_name|
+                        if !manifest.ignored?(pkg_name) && !manifest.excluded?(pkg_name)
+                            pkg.depends_on pkg_name
+                        end
+                    end
+                    pkg.os_packages.merge(osdeps)
+                    all_enabled_osdeps |= pkg.os_packages
+
+                    pkg.prepare if !pkg.disabled?
+                    Rake::Task["#{pkg.name}-prepare"].instance_variable_set(:@already_invoked, true)
+
+                    if options[:recursive]
+                        package_queue.concat(pkg.dependencies)
+                    end
+                end
+
+                if Autoproj.verbose
+                    Autoproj.message "autoproj: finished importing packages"
+                end
+
+                if options[:warn_about_excluded_packages]
+                    selection.exclusions.each do |sel, pkg_names|
+                        pkg_names.sort.each do |pkg_name|
+                            Autoproj.warn "#{pkg_name}, which was selected for #{sel}, cannot be built: #{manifest.exclusion_reason(pkg_name)}", :bold
+                        end
+                    end
+                end
+                if options[:warn_about_ignored_packages]
+                    selection.ignores.each do |sel, pkg_names|
+                        pkg_names.sort.each do |pkg_name|
+                            Autoproj.warn "#{pkg_name}, which was selected for #{sel}, is ignored", :bold
+                        end
+                    end
+                end
+
+                return all_enabled_sources, all_enabled_osdeps
+
+            ensure
+                if ws.config.import_log_enabled? && !updated_packages.empty? && Autoproj::Ops::Snapshot.update_log_available?(manifest)
+                    failure_message =
+                        if $!
+                            " (#{$!.message.split("\n").first})"
+                        end
+                    ops = Ops::Snapshot.new(ws.manifest, keep_going: true)
+                    ops.update_package_import_state(
+                        "#{$0} #{ARGV.join(" ")}#{failure_message}",
+                        updated_packages)
+                end
+            end
+        end
+    end
+end
+
+