autoproj 2.0.0.b5 → 2.0.0.b6

data/lib/autoproj/cli/switch_config.rb

@@ -11,15 +11,14 @@ module Autoproj
                     raise ConfigError, "you cannot run autoproj switch-config from autoproj's configuration directory or one of its subdirectories"
                 end
 
+                ws.load_config
+
                 # We must switch to the root dir first, as it is required by the
                 # configuration switch code. This is acceptable as long as we
                 # quit just after the switch
                 switcher = Ops::MainConfigSwitcher.new(ws)
                 if switcher.switch_config(*args)
-                    manifest = Manifest.load(File.join(ws.config_dir, 'manifest'))
-                    update = Ops::Configuration.new(ws, ws.loader)
-                    update.update_configuration
-                    ws.config.save
+                    CLI::Main.start(['update', '--config'])
                 end
             end
         end

data/lib/autoproj/cli/update.rb

@@ -21,10 +21,20 @@ module Autoproj
                     packages = ['.']
                 end
 
+                if options[:force_reset]
+                    options[:reset] = :force
+                end
+
                 if options[:autoproj].nil?
                     options[:autoproj] = packages.empty?
                 end
 
+                if mainline = options[:mainline]
+                    if mainline == 'mainline' || mainline == 'true'
+                        options[:mainline] = true
+                    end
+                end
+
                 return packages, options
             end
 
@@ -32,15 +42,12 @@ module Autoproj
                 selected_packages, config_selected =
                     normalize_command_line_package_selection(selected_packages)
 
-                if options[:apply_post_install].nil?
-                    options[:apply_post_install] = true
-                end
-
                 if options[:config].nil?
                     options[:config] = selected_packages.empty? || config_selected
                 end
 
                 ws.setup
+                parallel = options[:parallel] || ws.config.parallel_import_level
 
                 # Do that AFTER we have properly setup ws.osdeps as to avoid
                 # unnecessarily redetecting the operating system
@@ -56,11 +63,14 @@ module Autoproj
                 end
 
                 ws.load_package_sets(
+                    mainline: options[:mainline],
                     only_local: options[:local],
                     checkout_only: !options[:config] || options[:checkout_only],
-                    ignore_errors: options[:keep_going])
-                if selected_packages.empty? && config_selected
-                    return
+                    reset: options[:reset],
+                    ignore_errors: options[:keep_going],
+                    retry_count: options[:retry_count])
+                if selected_packages.empty? && (config_selected || options[:config]) && !options[:all]
+                    return [], [], true
                 end
 
                 ws.setup_all_package_directories
@@ -112,12 +122,11 @@ module Autoproj
                                     only_local: options[:local],
                                     reset: options[:reset],
                                     recursive: options[:deps],
-                                    ignore_errors: options[:keep_going])
+                                    ignore_errors: options[:keep_going],
+                                    parallel: parallel,
+                                    retry_count: options[:retry_count])
 
                 ws.finalize_setup
-                if options[:apply_post_install]
-                    ws.apply_post_install
-                end
                 ws.export_installation_manifest
 
                 if options[:osdeps] && !osdep_packages.empty?

data/lib/autoproj/cli/versions.rb

@@ -14,11 +14,11 @@ module Autoproj
 
             def validate_options(packages, options = Hash.new)
                 packages, options = super
-                if options[:save].nil?
+                if !options[:save].nil?
                     options[:save] = case options[:save]
                                      when '.'
                                          nil
-                                     when true
+                                     when 'save'
                                          default_versions_file
                                      else
                                          options[:save].to_str
@@ -29,21 +29,23 @@ module Autoproj
 
             def run(user_selection, options)
                 initialize_and_load
-                packages, *, config_selected =
+                user_selection, config_selected =
+                    normalize_command_line_package_selection(user_selection)
+                packages, * =
                     finalize_setup(user_selection,
                                    ignore_non_imported_packages: true)
 
-                if (config_selected || user_selection.empty?) && (options[:package_sets] != false)
-                    options[:package_sets] = true
-                end
-
-                ops = Ops::Snapshot.new(ws.manifest, keep_going: options[:keep_going])
+                
+                ops = Ops::Snapshot.new(ws.manifest, ignore_errors: options[:keep_going])
 
                 versions = Array.new
-                if options[:package_sets]
+                if config_selected || (options[:config] != false) || user_selection.empty?
                     versions += ops.snapshot_package_sets
                 end
-                versions += ops.snapshot_packages(packages)
+                if (!config_selected && !options[:config]) || !user_selection.empty?
+                    versions += ops.snapshot_packages(packages)
+                end
+
                 if output_file = options[:save]
                     ops.save_versions(versions, output_file, replace: options[:replace])
                 else

data/lib/autoproj/configuration.rb

@@ -189,6 +189,14 @@ module Autoproj
             set('import_log_enabled', !!value)
         end
 
+        def parallel_import_level
+            get('parallel_import_level', 10)
+        end
+
+        def parallel_import_level=(level)
+            set('parallel_import_level', level)
+        end
+
         def ruby_executable
             if path = get('ruby_executable', nil)
                 path

data/lib/autoproj/gitorious.rb

@@ -1,105 +1,3 @@
-module Autoproj
-    # Adds the relevant options to handle a gitorious server
-    # What this does is ask the user how he would like to access the gitorious
-    # server. Then, it sets
-    #
-    #   #{name}_ROOT to be the base URL for pulling
-    #   #{name}_PUSH_ROOT to be the corresponding ssh-based URL for pushing
-    #
-    # For instance, with
-    #
-    #   handle_gitorious_server "GITORIOUS", "gitorious.org"
-    #
-    # the URLs for the rtt repository in the orocos-toolchain gitorious project would
-    # be defined in the source.yml files as
-    #
-    #   ${GITORIOUS_ROOT}/orocos-toolchain/rtt.git
-    #   ${GITORIOUS_PUSH_ROOT}/orocos-toolchain/rtt.git
-    #
-    # Since it seems that the http method for gitorious servers is more stable, a
-    # fallback importer is set up that falls back to using http for pulling as soon
-    # as import failed
-    def self.gitorious_server_configuration(name, base_url, options = Hash.new)
-        options = Kernel.validate_options options,
-            :git_url  => "git://#{base_url}",
-            :http_url => "https://git.#{base_url}",
-            :ssh_url  => "git@#{base_url}:",
-            :fallback_to_http => true,
-            :default => 'http,ssh',
-            :disabled_methods => [],
-            config: Autoproj.config
-
-        config = options.delete(:config)
-        disabled_methods = Array(options[:disabled_methods])
-
-        access_methods = Hash[
-            'git' => 'git,ssh',
-            'ssh' => 'ssh,ssh',
-            'http' => 'http,http']
-
-        gitorious_long_doc = [
-            "How should I interact with #{base_url} (#{(access_methods.keys - disabled_methods).sort.join(", ")})",
-            "If you give two values, comma-separated, the first one will be",
-            "used for pulling and the second one for pushing"]
-
-        validator = lambda do |value|
-            if value =~ /,/
-                value.split(',').each do |method|
-                    if !access_methods.has_key?(method)
-                        raise Autoproj::InputError, "#{method} is not a known access method"
-                    elsif disabled_methods.include?(method)
-                        raise Autoproj::InputError, "#{method} is disabled on #{base_url}"
-                    end
-                end
-            elsif !access_methods.has_key?(value)
-                raise Autoproj::InputError, "#{value} is not a known access method"
-            elsif disabled_methods.include?(value)
-                raise Autoproj::InputError, "#{value} is disabled on #{base_url}"
-            end
-
-            value
-        end
-
-        config.declare name, 'string',
-            :default => options[:default],
-            :doc => gitorious_long_doc, &validator
-
-        access_mode = config.get(name)
-        begin
-            validator[access_mode]
-        rescue Autoproj::InputError => e
-            Autoproj.warn e.message
-            config.reset(name)
-            access_mode = config.get(name)
-        end
-        access_mode = access_methods[access_mode] || access_mode
-        pull, push = access_mode.split(',')
-        [[pull, "_ROOT"], [push, "_PUSH_ROOT"]].each do |method, var_suffix|
-            url = if method == "git" then options[:git_url]
-                  elsif method == "http" then options[:http_url]
-                  elsif method == "ssh" then options[:ssh_url]
-                  end
-            config.set("#{name}#{var_suffix}", url)
-        end
-
-        Autoproj.add_source_handler name.downcase do |url, vcs_options|
-            if url !~ /\.git$/
-                url += ".git"
-            end
-            if url !~ /^\//
-                url = "/#{url}"
-            end
-            pull_base_url = config.get("#{name}_ROOT")
-            push_base_url = config.get("#{name}_PUSH_ROOT")
-            Hash[type: 'git',
-                 url: "#{pull_base_url}#{url}",
-                 push_to: "#{push_base_url}#{url}",
-                 retry_count: 10,
-                 repository_id: "#{name.downcase}:#{url}"].merge(vcs_options)
-        end
-    end
-end
-
-Autoproj.gitorious_server_configuration('GITORIOUS', 'gitorious.org', :default => 'http,ssh', :disabled_methods => 'git')
-Autoproj.gitorious_server_configuration('GITHUB', 'github.com', :http_url => 'https://github.com', :default => 'http,ssh')
-
+Autoproj.warn "gitorious_server_configuration is deprecated, use require 'git_server_configuration' and Autoproj.git_server_configuration instead"
+Autoproj.warn "note that the method call interface has not changed, you just have to change the name(s)"
+require 'autoproj/git_server_configuration'

data/lib/autoproj/installation_manifest.rb

@@ -66,7 +66,7 @@ module Autoproj
         end
 
         def each(&block)
-            packages.each(&block)
+            packages.each_value(&block)
         end
 
         def [](name)

data/lib/autoproj/ops/configuration.rb

@@ -56,7 +56,9 @@ module Autoproj
             options = Kernel.validate_options options,
                 only_local: false,
                 checkout_only: !Autobuild.do_update,
-                ignore_errors: false
+                ignore_errors: false,
+                reset: false,
+                retry_count: nil
 
             fake_package = Tools.create_autobuild_package(vcs, name, into)
             if update_from
@@ -79,6 +81,9 @@ module Autoproj
                     fake_package.update = false
                 end
             end
+            if retry_count = options.delete(:retry_count)
+                fake_package.importer.retry_count = retry_count
+            end
             fake_package.import(options)
 
         rescue Autobuild::ConfigException => e
@@ -96,7 +101,9 @@ module Autoproj
             options = validate_options options,
                 only_local: false,
                 checkout_only: !Autobuild.do_update,
-                ignore_errors: false
+                ignore_errors: false,
+                reset: false,
+                retry_count: nil
 
             update_configuration_repository(
                 ws.manifest.vcs,
@@ -118,7 +125,9 @@ module Autoproj
             options = validate_options options,
                 only_local: false,
                 checkout_only: !Autobuild.do_update,
-                ignore_errors: false
+                ignore_errors: false,
+                reset: false,
+                retry_count: nil
 
             name = PackageSet.name_of(ws.manifest, vcs)
             raw_local_dir = PackageSet.raw_local_dir_of(vcs)
@@ -208,7 +217,9 @@ module Autoproj
             options = validate_options options,
                 only_local: false,
                 checkout_only: !Autobuild.do_update,
-                ignore_errors: false
+                ignore_errors: false,
+                reset: false,
+                retry_count: nil
 
             package_sets = [root_pkg_set]
             by_repository_id = Hash.new
@@ -339,7 +350,9 @@ module Autoproj
             options = validate_options options,
                 only_local: false,
                 checkout_only: true,
-                ignore_errors: false
+                ignore_errors: false,
+                reset: false,
+                retry_count: nil
             update_configuration(options)
         end
 
@@ -350,7 +363,9 @@ module Autoproj
             options = validate_options options,
                 only_local: false,
                 checkout_only: !Autobuild.do_update,
-                ignore_errors: false
+                ignore_errors: false,
+                reset: false,
+                retry_count: nil
 
             # Load the installation's manifest a first time, to check if we should
             # update it ... We assume that the OS dependencies for this VCS is already

data/lib/autoproj/ops/import.rb

@@ -63,12 +63,65 @@ module Autoproj
                 new_packages
             end
             
+            def pre_package_import(selection, manifest, pkg, reverse_dependencies)
+                # Try to auto-exclude the package early. If the autobuild file
+                # contained some information that allows us to exclude it now,
+                # then let's just do it
+                import_next_step(pkg, reverse_dependencies)
+                if manifest.excluded?(pkg.name)
+                    selection.filter_excluded_and_ignored_packages(manifest)
+                    false
+                elsif manifest.ignored?(pkg.name)
+                    false
+                elsif !pkg.importer && !File.directory?(pkg.srcdir)
+                    raise ConfigError.new, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
+                elsif pkg.importer
+                    true
+                end
+            end
+
+            def post_package_import(selection, manifest, pkg, reverse_dependencies)
+                Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
+                manifest.load_package_manifest(pkg.name)
+
+                # The package setup mechanisms might have added an exclusion
+                # on this package. Handle this.
+                if manifest.excluded?(pkg.name)
+                    mark_exclusion_along_revdeps(pkg.name, reverse_dependencies)
+                    # Run a filter now, to have errors as early as possible
+                    selection.filter_excluded_and_ignored_packages(manifest)
+                    # Delete this package from the current_packages set
+                    false
+                elsif manifest.ignored?(pkg.name)
+                    false
+                else
+                    Autoproj.each_post_import_block(pkg) do |block|
+                        block.call(pkg)
+                    end
+                    import_next_step(pkg, reverse_dependencies)
+                end
+            end
+
+            class ImportFailed < RuntimeError; end
+
             # Import all packages from the given selection, and their
             # dependencies
             def import_selected_packages(selection, updated_packages, options = Hash.new)
+                all_processed_packages = Set.new
+
+                parallel_options, options = Kernel.filter_options options,
+                    parallel: ws.config.parallel_import_level
+
+                # This is used in the ensure block, initialize as early as
+                # possible
+                executor = Concurrent::FixedThreadPool.new(parallel_options[:parallel], max_length: 0)
+
                 options, import_options = Kernel.filter_options options,
-                    recursive: true
+                    recursive: true,
+                    retry_count: nil
 
+                ignore_errors = options[:ignore_errors]
+                retry_count = options[:retry_count]
                 manifest = ws.manifest
 
                 selected_packages = selection.each_source_package_name.map do |pkg_name|
@@ -83,72 +136,107 @@ module Autoproj
                 # package exclusion (and that does not affect optional dependencies)
                 reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }
 
+                completion_queue = Queue.new
+                pending_packages = Set.new
                 # The set of all packages that are currently selected by +selection+
                 all_processed_packages = Set.new
+                interactive_imports = Array.new
                 package_queue = selected_packages.to_a.sort_by(&:name)
-                while !package_queue.empty?
-                    pkg = package_queue.shift
-                    # Remove packages that have already been processed
-                    next if all_processed_packages.include?(pkg.name)
-                    all_processed_packages << pkg.name
-
-                    # Try to auto-exclude the package early. If the autobuild file
-                    # contained some information that allows us to exclude it now,
-                    # then let's just do it
-                    import_next_step(pkg, reverse_dependencies)
-                    if manifest.excluded?(pkg.name)
-                        selection.filter_excluded_and_ignored_packages(manifest)
-                        next
-                    elsif manifest.ignored?(pkg.name)
-                        next
-                    end
+                failures = Hash.new
+                while failures.empty? || ignore_errors
+                    # Queue work for all packages in the queue
+                    package_queue.each do |pkg|
+                        # Remove packages that have already been processed
+                        next if all_processed_packages.include?(pkg)
+                        all_processed_packages << pkg
 
-                    # If the package has no importer, the source directory must
-                    # be there
-                    if !pkg.importer && !File.directory?(pkg.srcdir)
-                        raise ConfigError.new, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
-                    end
+                        if !pre_package_import(selection, manifest, pkg, reverse_dependencies)
+                            next
+                        elsif pkg.importer.interactive?
+                            interactive_imports << pkg
+                            next
+                        end
 
-                    ## COMPLETELY BYPASS RAKE HERE
-                    # The reason is that the ordering of import/prepare between
-                    # packages is not important BUT the ordering of import vs.
-                    # prepare in one package IS important: prepare is the method
-                    # that takes into account dependencies.
-                    pkg.import(import_options)
-                    if pkg.updated?
-                        updated_packages << pkg.name
-                    end
-                    Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
-                    manifest.load_package_manifest(pkg.name)
-
-                    # The package setup mechanisms might have added an exclusion
-                    # on this package. Handle this.
-                    if manifest.excluded?(pkg.name)
-                        mark_exclusion_along_revdeps(pkg.name, reverse_dependencies)
-                        # Run a filter now, to have errors as early as possible
-                        selection.filter_excluded_and_ignored_packages(manifest)
-                        # Delete this package from the current_packages set
-                        next
-                    elsif manifest.ignored?(pkg.name)
-                        next
+                        pending_packages << pkg
+                        import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
+                            ## COMPLETELY BYPASS RAKE HERE
+                            # The reason is that the ordering of import/prepare between
+                            # packages is not important BUT the ordering of import vs.
+                            # prepare in one package IS important: prepare is the method
+                            # that takes into account dependencies.
+                            if retry_count
+                                import_pkg.importer.retry_count = retry_count
+                            end
+                            import_pkg.import(import_options.merge(allow_interactive: false))
+                        end
+                        import_future.add_observer do |time, result, reason|
+                            completion_queue << [pkg, time, result, reason]
+                        end
+                        import_future.execute
                     end
+                    package_queue.clear
 
-                    Autoproj.each_post_import_block(pkg) do |block|
-                        block.call(pkg)
+                    if completion_queue.empty? && pending_packages.empty?
+                        # We've nothing to process anymore ... process
+                        # interactive imports if there are some. Otherwise,
+                        # we're done
+                        if interactive_imports.empty?
+                            return all_processed_packages
+                        else
+                            interactive_imports.each do |pkg|
+                                begin
+                                    result = pkg.import(import_options.merge(allow_interactive: true))
+                                rescue Exception => reason
+                                end
+                                completion_queue << [pkg, Time.now, result, reason]
+                            end
+                            interactive_imports.clear
+                        end
                     end
 
-                    new_packages = import_next_step(pkg, reverse_dependencies)
-
-                    # Excluded dependencies might have caused the package to be
-                    # excluded as well ... do not add any dependency to the
-                    # processing queue if it is the case
-                    if manifest.excluded?(pkg.name)
-                        selection.filter_excluded_and_ignored_packages(manifest)
-                    elsif options[:recursive]
-                        package_queue.concat(new_packages.sort_by(&:name))
+                    # And wait one to finish
+                    pkg, time, result, reason = completion_queue.pop
+                    pending_packages.delete(pkg)
+                    if reason
+                        if reason.kind_of?(Autobuild::InteractionRequired)
+                            interactive_imports << pkg
+                        else
+                            # One importer failed... terminate
+                            Autoproj.error "import of #{pkg.name} failed"
+                            if !reason.kind_of?(Interrupt)
+                                Autoproj.error "#{reason}"
+                            end
+                            failures[pkg] = reason
+                        end
+                    else
+                        if new_packages = post_package_import(selection, manifest, pkg, reverse_dependencies)
+                            # Excluded dependencies might have caused the package to be
+                            # excluded as well ... do not add any dependency to the
+                            # processing queue if it is the case
+                            if manifest.excluded?(pkg.name)
+                                selection.filter_excluded_and_ignored_packages(manifest)
+                            elsif options[:recursive]
+                                package_queue = new_packages.sort_by(&:name)
+                            end
+                        end
                     end
                 end
+
+                if !failures.empty?
+                    raise ImportFailed, "import of #{failures.size} packages failed: #{failures.keys.map(&:name).sort.join(", ")}"
+                end
+
                 all_processed_packages
+
+            ensure
+                if failures && !failures.empty? && !ignore_errors
+                    Autoproj.error "waiting for pending import jobs to finish"
+                end
+                if executor
+                    executor.shutdown
+                    executor.wait_for_termination
+                end
+                updated_packages.concat(all_processed_packages.find_all(&:updated?).map(&:name))
             end
             
             def finalize_package_load(processed_packages)
@@ -164,7 +252,7 @@ module Autoproj
                     next if manifest.ignored?(pkg_name) || manifest.excluded?(pkg_name)
 
                     pkg = manifest.find_autobuild_package(pkg_name)
-                    if !processed_packages.include?(pkg.name)
+                    if !processed_packages.include?(pkg)
                         manifest.load_package_manifest(pkg.name)
                         Autoproj.each_post_import_block(pkg) do |block|
                             block.call(pkg)
@@ -187,6 +275,9 @@ module Autoproj
             end
 
             def import_packages(selection, options = Hash.new)
+                # Used in the ensure block, initialize as soon as possible
+                updated_packages = Array.new
+
                 options, import_options = Kernel.filter_options options,
                     warn_about_ignored_packages: true,
                     warn_about_excluded_packages: true,
@@ -194,13 +285,17 @@ module Autoproj
 
                 manifest = ws.manifest
 
-                updated_packages = Array.new
                 all_processed_packages = import_selected_packages(
                     selection, updated_packages, import_options.merge(recursive: options[:recursive]))
                 finalize_package_load(all_processed_packages)
 
                 all_enabled_osdeps = selection.each_osdep_package_name.to_set
-                all_enabled_sources = all_processed_packages
+                all_enabled_sources = all_processed_packages.map(&:name)
+                if options[:recursive]
+                    all_processed_packages.each do |pkg|
+                        all_enabled_osdeps.merge(pkg.os_packages)
+                    end
+                end
 
                 if options[:warn_about_excluded_packages]
                     selection.exclusions.each do |sel, pkg_names|
@@ -225,7 +320,7 @@ module Autoproj
                         if $!
                             " (#{$!.message.split("\n").first})"
                         end
-                    ops = Ops::Snapshot.new(ws.manifest, keep_going: true)
+                    ops = Ops::Snapshot.new(ws.manifest, ignore_errors: true)
                     ops.update_package_import_state(
                         "#{$0} #{ARGV.join(" ")}#{failure_message}",
                         updated_packages)