autoproj 2.0.0.rc38 → 2.0.0.rc39

data/lib/autoproj/ops/import.rb

@@ -149,15 +149,13 @@ def queue_import_work(executor, completion_queue, pkg, retry_count: nil, **impor
                 import_future.execute
             end
 
-            class ImportFailed < RuntimeError; end
-
             # Import all packages from the given selection, and their
             # dependencies
-            def import_selected_packages(selection, updated_packages,
+            def import_selected_packages(selection,
                                          parallel: ws.config.parallel_import_level,
                                          recursive: true,
                                          retry_count: nil,
-                                         ignore_errors: false,
+                                         keep_going: false,
                                          install_vcs_packages: Hash.new,
                                          non_imported_packages: :checkout,
                                          **import_options)
@@ -189,10 +187,10 @@ def import_selected_packages(selection, updated_packages,
                 all_processed_packages = Set.new
                 main_thread_imports = Array.new
                 package_queue = selected_packages.to_a.sort_by(&:name)
-                failures = Hash.new
+                failures = Array.new
                 missing_vcs = Array.new
                 installed_vcs_packages = Set['none', 'local']
-                while failures.empty? || ignore_errors
+                while failures.empty? || keep_going
                     # Queue work for all packages in the queue
                     package_queue.each do |pkg|
                         # Remove packages that have already been processed
@@ -283,7 +281,7 @@ def import_selected_packages(selection, updated_packages,
                             if !reason.kind_of?(Interrupt)
                                 Autoproj.error "#{reason}"
                             end
-                            failures[pkg] = reason
+                            failures << reason
                         end
                     else
                         if new_packages = post_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
@@ -299,29 +297,19 @@ def import_selected_packages(selection, updated_packages,
                     end
                 end
 
-                if !failures.empty?
-                    raise ImportFailed, "import of #{failures.size} packages failed: #{failures.keys.map(&:name).sort.join(", ")}"
-                end
-
                 all_processed_packages.delete_if do |processed_pkg|
                     ws.manifest.excluded?(processed_pkg.name) || ws.manifest.ignored?(processed_pkg.name)
                 end
-                all_processed_packages
+                return all_processed_packages, failures
 
             ensure
-                if failures && !failures.empty? && !ignore_errors
+                if failures && !failures.empty? && !keep_going
                     Autoproj.error "waiting for pending import jobs to finish"
                 end
                 if executor
                     executor.shutdown
                     executor.wait_for_termination
                 end
-                if all_processed_packages
-                    all_updated_packages = all_processed_packages.find_all do |processed_pkg|
-                        processed_pkg.autobuild.updated?
-                    end
-                    updated_packages.concat(all_updated_packages.map(&:name))
-                end
             end
             
             def finalize_package_load(processed_packages)
@@ -373,22 +361,24 @@ def import_packages(selection,
                                 warn_about_ignored_packages: true,
                                 warn_about_excluded_packages: true,
                                 recursive: true,
-                                ignore_errors: false,
+                                keep_going: false,
                                 install_vcs_packages: Hash.new,
                                 **import_options)
 
-                # Used in the ensure block, initialize as soon as possible
-                updated_packages = Array.new
-
                 manifest = ws.manifest
 
-                all_processed_packages = import_selected_packages(
-                    selection, updated_packages,
+                all_processed_packages, failures = import_selected_packages(
+                    selection,
                     non_imported_packages: non_imported_packages,
-                    ignore_errors: ignore_errors,
+                    keep_going: keep_going,
                     recursive: recursive,
                     install_vcs_packages: install_vcs_packages,
                     **import_options)
+
+                if !keep_going && !failures.empty?
+                    raise failures.first
+                end
+
                 finalize_package_load(all_processed_packages)
 
                 all_enabled_osdeps = selection.each_osdep_package_name.to_set
@@ -414,18 +404,34 @@ def import_packages(selection,
                     end
                 end
 
-                return all_enabled_sources, all_enabled_osdeps
+                if !failures.empty?
+                    raise PackageImportFailed.new(
+                        failures, source_packages: all_enabled_sources,
+                        osdep_packages: all_enabled_osdeps)
+                else
+                    return all_enabled_sources, all_enabled_osdeps
+                end
 
             ensure
-                if ws.config.import_log_enabled? && !updated_packages.empty? && Autoproj::Ops::Snapshot.update_log_available?(manifest)
+                if ws.config.import_log_enabled? && Autoproj::Ops::Snapshot.update_log_available?(manifest)
+                    update_log_for_processed_packages(all_processed_packages || Array.new, $!)
+                end
+            end
+
+            def update_log_for_processed_packages(all_processed_packages, exception)
+                all_updated_packages = all_processed_packages.find_all do |processed_pkg|
+                    processed_pkg.autobuild.updated?
+                end
+
+                if !all_updated_packages.empty?
                     failure_message =
-                        if $!
-                            " (#{$!.message.split("\n").first})"
+                        if exception
+                            " (#{exception.message.split("\n").first})"
                         end
-                    ops = Ops::Snapshot.new(ws.manifest, ignore_errors: true)
+                    ops = Ops::Snapshot.new(ws.manifest, keep_going: true)
                     ops.update_package_import_state(
                         "#{$0} #{ARGV.join(" ")}#{failure_message}",
-                        updated_packages)
+                        all_updated_packages.map(&:name))
                 end
             end
         end

data/lib/autoproj/ops/main_config_switcher.rb

@@ -127,7 +127,7 @@ def switch_config(*args)
 
                 url = VCSDefinition.to_absolute_url(url, ws.root_dir)
 
-                if vcs && (vcs.type == type && vcs.url == url)
+                if vcs.type == type && vcs.url == url
                     vcs = vcs.to_hash
                     options.each do |opt|
                         opt_name, opt_value = opt.split('=')

data/lib/autoproj/ops/snapshot.rb

@@ -43,12 +43,9 @@ def sort_versions(versions)
                 pkgs.sort_by { |vcs| vcs.keys.first }
         end
 
-        def save_versions( versions, versions_file, options = Hash.new )
-            options = Kernel.validate_options options,
-                replace: false
-
+        def save_versions( versions, versions_file, replace: false)
             existing_versions = Array.new
-            if !options[:replace] && File.exist?(versions_file)
+            if !replace && File.exist?(versions_file)
                 existing_versions = YAML.load( File.read( versions_file ) ) ||
                     Array.new
             end
@@ -80,25 +77,20 @@ def self.snapshot( packages, target_dir )
         #
         # @return [Boolean]
         # @see initialize error_or_warn
-        def ignore_errors?; !!@ignore_errors end
+        def keep_going?; !!@keep_going end
 
-        def initialize(manifest, options = Hash.new)
+        def initialize(manifest, keep_going: false)
             @manifest = manifest
-            options = Kernel.validate_options options,
-                ignore_errors: false
-            @ignore_errors = options[:ignore_errors]
+            @keep_going = keep_going
         end
 
-        def snapshot_package_sets(target_dir = nil, options = Hash.new)
-            options = Kernel.validate_options options,
-                only_local: true
-
+        def snapshot_package_sets(target_dir = nil, only_local: true)
             result = Array.new
             manifest.each_package_set do |pkg_set|
                 next if pkg_set.local?
 
                 vcs_info =
-                    begin pkg_set.snapshot(target_dir, only_local: options[:only_local])
+                    begin pkg_set.snapshot(target_dir, only_local: only_local)
                     rescue Exception => e
                         error_or_warn(pkg_set, e)
                         next
@@ -116,7 +108,7 @@ def snapshot_package_sets(target_dir = nil, options = Hash.new)
         def error_or_warn(package, error)
             if error.kind_of?(Interrupt)
                 raise
-            elsif ignore_errors?
+            elsif keep_going?
                 if !error.respond_to?(:to_str)
                     error = error.message
                 end
@@ -128,10 +120,7 @@ def error_or_warn(package, error)
             end
         end
 
-        def snapshot_packages(packages, target_dir = nil, options = Hash.new)
-            options = Kernel.validate_options options,
-                only_local: true
-
+        def snapshot_packages(packages, target_dir = nil, only_local: true)
             result = Array.new
             packages.each do |package_name|
                 package  = manifest.find_package_definition(package_name)
@@ -148,7 +137,7 @@ def snapshot_packages(packages, target_dir = nil, options = Hash.new)
                 end
 
                 vcs_info =
-                    begin importer.snapshot(package.autobuild, target_dir, only_local: options[:only_local])
+                    begin importer.snapshot(package.autobuild, target_dir, only_local: only_local)
                     rescue Exception => e
                         error_or_warn(package, e)
                         next

data/lib/autoproj/os_package_installer.rb

@@ -50,6 +50,7 @@ def initialize(ws, os_package_resolver, package_managers: PACKAGE_MANAGERS)
             @installed_resolved_packages = Hash.new { |h, k| h[k] = Set.new }
             @silent = true
             @filter_uptodate_packages = true
+            @osdeps_mode = nil
 
             @package_managers = Hash.new
             package_managers.each do |name, klass|

data/lib/autoproj/os_package_resolver.rb

@@ -134,7 +134,7 @@ def prefer_indep_over_os_packages=(flag); @prefer_indep_over_os_packages = flag
         # Use to override the autodetected OS-specific package handler
         def os_package_manager=(manager_name)
             if manager_name && !package_managers.include?(manager_name)
-                raise ArgumentError, "#{manager_name} is not a known package manager"
+                raise ArgumentError, "#{manager_name} is not a known package manager, known managers are #{package_managers.to_a.sort.join(", ")}"
             end
             @os_package_manager = manager_name
         end
@@ -214,8 +214,6 @@ def invalidate_resolve_package_cache
         # defined in both OSPackageResolver objects, the information in +info+
         # takes precedence
         def merge(info)
-            invalidate_resolve_package_cache
-
             @definitions = definitions.merge(info.definitions) do |h, v1, v2|
                 if v1 != v2
                     old = source_of(h)
@@ -224,11 +222,11 @@ def merge(info)
                     # Warn if the new osdep definition resolves to a different
                     # set of packages than the old one
                     old_resolved = resolve_package(h).inject(Hash.new) do |osdep_h, (handler, status, list)|
-                        osdep_h[handler] = [status, list]
+                        osdep_h[handler] = [status, list.dup]
                         osdep_h
                     end
                     new_resolved = info.resolve_package(h).inject(Hash.new) do |osdep_h, (handler, status, list)|
-                        osdep_h[handler] = [status, list]
+                        osdep_h[handler] = [status, list.dup]
                         osdep_h
                     end
                     if old_resolved != new_resolved
@@ -237,6 +235,8 @@ def merge(info)
                 end
                 v2
             end
+            invalidate_resolve_package_cache
+
             @sources = sources.merge(info.sources)
             @all_definitions = all_definitions.merge(info.all_definitions) do |package_name, all_defs, new_all_defs|
                 all_defs = all_defs.dup
@@ -270,6 +270,12 @@ def self.verify_definitions(hash, path = [])
             end
         end
 
+        # Whether the operating system could be autodetected successfully
+        def known_operating_system?
+            os_names, _ = operating_system
+            !os_names.empty?
+        end
+
         # Returns true if it is possible to install packages for the operating
         # system on which we are installed
         def supported_operating_system?
@@ -497,26 +503,27 @@ def resolve_package(name)
             path = self.class.resolve_name(name)
             name = path.last
 
-            os_names, os_versions = operating_system
-            os_names = os_names.dup
-            if prefer_indep_over_os_packages?
-                os_names.unshift 'default'
-            else
-                os_names.push 'default'
-            end
-
             dep_def = definitions[name]
             if !dep_def
                 return (resolve_package_cache[name] = nil)
             end
 
+            os_names, os_versions = operating_system
+
             # Partition the found definition in all entries that are interesting
             # for us: toplevel os-independent package managers, os-dependent
             # package managers and os-independent package managers selected by
             # OS or version
-            if !os_names
+            if os_names.empty?
                 os_names = ['default']
                 os_versions = ['default']
+            else
+                os_names = os_names.dup
+                if prefer_indep_over_os_packages?
+                    os_names.unshift 'default'
+                else
+                    os_names.push 'default'
+                end
             end
 
             result = []
@@ -545,6 +552,10 @@ def resolve_package(name)
                 end
             end
 
+            result.each do |args|
+                args.last.freeze
+            end
+            result.freeze
             resolve_package_cache[name] = result
         end
 
@@ -714,7 +725,11 @@ def resolve_os_packages(dependencies)
 
                 if result.empty?
                     os_names, os_versions = operating_system
-                    raise MissingOSDep.new, "there is an osdeps definition for #{name}, but not for this operating system and version (resp. #{os_names.join(", ")} and #{os_versions.join(", ")})"
+                    if os_names.empty?
+                        raise MissingOSDep.new, "there is an osdeps definition for #{name}, but autoproj cannot detect the local operation system"
+                    else
+                        raise MissingOSDep.new, "there is an osdeps definition for #{name}, but not for this operating system and version (resp. #{os_names.join(", ")} and #{os_versions.join(", ")})"
+                    end
                 end
 
                 result.each do |handler, status, packages|
@@ -724,7 +739,7 @@ def resolve_os_packages(dependencies)
                     if entry = all_packages.find { |h, _| h == handler }
                         entry[1].concat(packages)
                     else
-                        all_packages << [handler, packages]
+                        all_packages << [handler, packages.dup]
                     end
                 end
             end
@@ -784,13 +799,14 @@ def availability_of(name)
             end
 
             if resolved.empty?
-                if !operating_system
+                if known_operating_system?
+                    return WRONG_OS
+                else
                     return UNKNOWN_OS
-                else return WRONG_OS
                 end
             end
 
-            resolved = resolved.delete_if { |_, status, list| status == FOUND_PACKAGES && list.empty? }
+            resolved = resolved.find_all { |_, status, list| status != FOUND_PACKAGES || !list.empty? }
             failed = resolved.find_all do |_, status, _|
                 status == FOUND_NONEXISTENT
             end

data/lib/autoproj/system.rb

@@ -32,6 +32,9 @@ def self.root_dir=(dir)
     def self.root_dir(dir = Dir.pwd)
         if @root_dir
             return @root_dir
+        elsif !dir
+            @root_dir = ni
+            return
         end
         path = Autoproj.find_workspace_dir(dir)
         if !path

data/lib/autoproj/test.rb

@@ -292,12 +292,29 @@ def ws_define_package_set(name, vcs = VCSDefinition.from_raw(type: 'none'), **op
             package_set
         end
 
+        def ws_create_local_package_set(name, path, source_data: Hash.new, **options)
+            vcs = VCSDefinition.from_raw(type: 'local', url: path)
+            package_set = PackageSet.new(ws, vcs, name: name, **options)
+            FileUtils.mkdir_p(path)
+            File.open(File.join(path, 'source.yml'), 'w') do |io|
+                YAML.dump(Hash['name' => name].merge(source_data), io)
+            end
+            ws.manifest.register_package_set(package_set)
+            package_set
+        end
+
         def ws_add_package_set_to_layout(name, vcs = VCSDefinition.from_raw(type: 'none'), **options)
             package_set = ws_define_package_set(name, vcs, **options)
             ws.manifest.add_package_set_to_layout(package_set)
             package_set
         end
 
+        def ws_add_metapackage_to_layout(name, *packages)
+            meta = ws.manifest.metapackage(name, *packages)
+            ws.manifest.add_metapackage_to_layout(meta)
+            meta
+        end
+
         def ws_define_osdep_entries(entries)
             ws_os_package_resolver.add_entries(entries)
         end

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "2.0.0.rc38"
+    VERSION = "2.0.0.rc39"
 end

data/lib/autoproj/workspace.rb

@@ -138,17 +138,9 @@ def config_file_path
             File.join(dot_autoproj_dir, 'config.yml')
         end
 
-        # The path to a workspace's manifest file given its root dir
-        #
-        # @param [String] root_dir the workspace root directory
-        # @return [String]
-        def self.manifest_file_path_for(root_dir)
-            File.join(root_dir, 'autoproj', 'manifest')
-        end
-
         # The path to the workspace's manifest file
         def manifest_file_path
-            self.class.manifest_file_path_for(root_dir)
+            File.join(root_dir, 'autoproj', config.get('manifest_name', 'manifest'))
         end
 
         # Return the directory in which remote package set definition should be
@@ -210,12 +202,6 @@ def save_config
             config.save(config_file_path)
         end
 
-        def load_manifest
-            if File.exist?(manifest_file_path)
-                manifest.load(manifest_file_path)
-            end
-        end
-
         def autodetect_operating_system(force: false)
             if force || !os_package_resolver.operating_system
                 begin
@@ -262,7 +248,9 @@ def setup
             config.each_reused_autoproj_installation do |p|
                 manifest.reuse(p)
             end
-            load_manifest
+            if File.exist?(manifest_file_path)
+                manifest.load(manifest_file_path)
+            end
 
             Autobuild.prefix = prefix_dir
             FileUtils.mkdir_p File.join(prefix_dir, '.autoproj')
@@ -386,6 +374,20 @@ def set_as_main_workspace
             Autoproj.workspace = self
             Autoproj.root_dir = root_dir
             Autobuild.env = env
+
+            if block_given?
+                begin
+                    yield
+                ensure 
+                    clear_main_workspace
+                end
+            end
+        end
+
+        def clear_main_workspace
+            Autoproj.workspace = nil
+            Autoproj.root_dir = nil
+            Autobuild.env = nil
         end
 
         # Loads autoproj/init.rb
@@ -418,127 +420,29 @@ def load_autoprojrc
             end
         end
 
-        # Load OS dependency information contained in our registered package
-        # sets into the provided osdep object
-        #
-        # This is included in {load_package_sets}
-        #
-        # @return [void]
-        def load_osdeps_from_package_sets
-            manifest.each_package_set do |pkg_set|
-                pkg_set.each_osdeps_file do |file|
-                    file_osdeps = pkg_set.load_osdeps(file, operating_system: os_package_resolver.operating_system)
-                    os_package_resolver.merge(file_osdeps)
-                end
-            end
-        end
-
-        def load_package_sets(options = Hash.new)
+        def load_package_sets(only_local: false,
+                              checkout_only: true,
+                              reconfigure: false,
+                              keep_going: false,
+                              mainline: nil,
+                              reset: false,
+                              retry_count: nil)
             if !File.file?(manifest_file_path) # empty install, just return
                 return
             end
 
-            options = validate_options options,
-                only_local: false,
-                checkout_only: true,
-                silent: false, # NOTE: this is ignored, enclose call with Autoproj.silent { }
-                reconfigure: false,
-                ignore_errors: false,
-                mainline: nil,
-                reset: false,
-                retry_count: nil
-
-            Ops::Configuration.new(self).
-                load_package_sets(only_local: options[:only_local],
-                                  checkout_only: options[:checkout_only],
-                                  ignore_errors: options[:ignore_errors],
-                                  reset: options[:reset],
-                                  retry_count: options[:retry_count])
-
-            manifest.each_package_set do |pkg_set|
-                if Gem::Version.new(pkg_set.required_autoproj_version) > Gem::Version.new(Autoproj::VERSION)
-                    raise ConfigError.new(pkg_set.source_file), "the #{pkg_set.name} package set requires autoproj v#{pkg_set.required_autoproj_version} but this is v#{Autoproj::VERSION}"
-                end
-            end
-
-            # Loads OS package definitions once and for all
-            load_osdeps_from_package_sets
-
-            # Load the required autobuild definitions
-            Autoproj.message("autoproj: loading ...", :bold)
-            if !options[:reconfigure]
+            if !reconfigure
                 Autoproj.message("run 'autoproj reconfigure' to change configuration options", :bold)
                 Autoproj.message("and use 'autoproj switch-config' to change the remote source for", :bold)
                 Autoproj.message("autoproj's main build configuration", :bold)
             end
-            manifest.each_package_set do |pkg_set|
-                pkg_set.each_autobuild_file do |path|
-                    import_autobuild_file pkg_set, path
-                end
-            end
-
-            # Now, load the package's importer configurations (from the various
-            # source.yml files)
-            mainline = options[:mainline]
-            if mainline.respond_to?(:to_str)
-                mainline = manifest.package_set(mainline)
-            end
-            manifest.load_importers(mainline: mainline)
-
-            # Auto-add packages that are
-            #  * present on disk
-            #  * listed in the layout part of the manifest
-            #  * but have no definition
-            explicit = manifest.normalized_layout
-            explicit.each do |pkg_or_set, layout_level|
-                next if manifest.find_autobuild_package(pkg_or_set)
-                next if manifest.has_package_set?(pkg_or_set)
-
-                # This is not known. Check if we can auto-add it
-                full_path = File.expand_path(File.join(root_dir, layout_level, pkg_or_set))
-                next if !File.directory?(full_path)
-
-                handler, _srcdir = Autoproj.package_handler_for(full_path)
-                if handler
-                    Autoproj.message "  auto-adding #{pkg_or_set} #{"in #{layout_level} " if layout_level != "/"}using the #{handler.gsub(/_package/, '')} package handler"
-                    in_package_set(manifest.main_package_set, manifest.file) do
-                        send(handler, pkg_or_set)
-                    end
-                else
-                    Autoproj.warn "cannot auto-add #{pkg_or_set}: unknown package type"
-                end
-            end
-
-            manifest.each_autobuild_package do |pkg|
-                Autobuild.each_utility do |uname, _|
-                    pkg.utility(uname).enabled =
-                        config.utility_enabled_for?(uname, pkg.name)
-                end
-            end
-
-            # And exclude any package that is not available on this particular
-            # configuration
-            mark_unavailable_osdeps_as_excluded
-        end
-
-        def mark_unavailable_osdeps_as_excluded
-            os_package_resolver.all_package_names.each do |osdep_name|
-                # If the osdep can be replaced by source packages, there's
-                # nothing to do really. The exclusions of the source packages
-                # will work as expected
-                if manifest.osdeps_overrides[osdep_name] || manifest.find_autobuild_package(osdep_name)
-                    next
-                end
-
-                case os_package_resolver.availability_of(osdep_name)
-                when OSPackageResolver::UNKNOWN_OS
-                    manifest.exclude_package(osdep_name, "this operating system is unknown to autoproj")
-                when OSPackageResolver::WRONG_OS
-                    manifest.exclude_package(osdep_name, "there are definitions for it, but not for this operating system")
-                when OSPackageResolver::NONEXISTENT
-                    manifest.exclude_package(osdep_name, "it is marked as unavailable for this operating system")
-                end
-            end
+            Ops::Configuration.new(self).
+                load_package_sets(only_local: only_local,
+                                  checkout_only: checkout_only,
+                                  keep_going: keep_going,
+                                  reset: reset,
+                                  retry_count: retry_count,
+                                  mainline: mainline)
         end
 
         def load_packages(selection = manifest.default_packages(false), options = Hash.new)
@@ -705,7 +609,7 @@ def all_os_packages(import_missing: false, parallel: config.parallel_import_leve
                 _, all_os_packages =
                     ops.import_packages(manifest.default_packages,
                                     checkout_only: true, only_local: true, reset: false,
-                                    recursive: true, ignore_errors: true, parallel: parallel,
+                                    recursive: true, keep_going: true, parallel: parallel,
                                     retry_count: 0)
                 all_os_packages
             else
@@ -761,7 +665,9 @@ def self.workspace
 
     def self.workspace=(ws)
         @workspace = ws
-        self.root_dir = ws.root_dir
+        if ws
+            self.root_dir = ws.root_dir
+        end
     end
 
     def self.env