autoproj 2.4.0 → 2.5.0.pre1

data/lib/autoproj/cli/main.rb

@@ -2,6 +2,7 @@
 require 'tty/color'
 require 'autoproj/cli/main_test'
 require 'autoproj/cli/main_plugin'
+require 'autoproj/reporter'
 
 module Autoproj
     module CLI
@@ -34,13 +35,13 @@ def default_report_on_package_failures
                     end
                 end
 
-                def run_autoproj_cli(filename, classname, report_options, *args, **extra_options)
+                def run_autoproj_cli(filename, classname, report_options, *args, tool_failure_mode: :exit_silent, **extra_options)
                     require "autoproj/cli/#{filename}"
                     if Autobuild::Subprocess.transparent_mode = options[:tool]
                         Autobuild.silent = true
                         Autobuild.color = false
                         report_options[:silent] = true
-                        report_options[:on_package_failures] = :exit_silent
+                        report_options[:on_package_failures] = tool_failure_mode
                         extra_options[:silent] = true
                     end
 
@@ -83,6 +84,13 @@ def envsh
                 run_autoproj_cli(:envsh, :Envsh, Hash[])
             end
 
+            desc 'watch', 'watch workspace for changes', hide: true
+            option :show_events, type: :boolean, default: false,
+                desc: "whether detected events should be displayed"
+            def watch
+                run_autoproj_cli(:watch, :Watch, Hash[])
+            end
+
             desc 'status [PACKAGES]', 'displays synchronization status between this workspace and the package(s) source'
             option :local, type: :boolean, default: false,
                 desc: 'only use locally available information (mainly for distributed version control systems such as git)'
@@ -195,7 +203,21 @@ def build(*packages)
                     report_options[:on_package_failures] = :report
                 end
 
-                run_autoproj_cli(:build, :Build, Hash[silent: false].merge(report_options), *packages)
+                failures = run_autoproj_cli(:build, :Build, report_options, *packages,
+                    tool_failure_mode: :report_silent)
+                if !failures.empty?
+                    Autobuild.silent = false
+                    packages_failed = failures.
+                        map do |e|
+                            if e.respond_to?(:target) && e.target.respond_to?(:name)
+                                e.target.name
+                            end
+                        end.compact
+                    if !packages_failed.empty?
+                        Autobuild.error "#{packages_failed.size} packages failed: #{packages_failed.sort.join(", ")}"
+                    end
+                    exit 1
+                end
             end
 
             desc 'cache CACHE_DIR', 'create or update a cache directory that can be given to AUTOBUILD_CACHE_DIR'
@@ -343,12 +365,12 @@ def tag(tag_name = nil, *packages)
                 run_autoproj_cli(:tag, :Tag, Hash[], tag_name, *packages)
             end
 
-            desc 'commit [PACKAGES]', 'save the package current versions as a new commit in the main build configuration'
+            desc 'commit [TAG_NAME] [PACKAGES]', 'save the package current versions as a new commit in the main build configuration'
             long_desc <<-EOD
 The commit subcommand stores the state of all packages (or of the packages
 selected on the command line) into a new commit in the currently checked-out
 branch of the build configuration. This state can be retrieved later on by using
-"autoproj reset"
+"autoproj reset". If a TAG_NAME is provided, the commit will be tagged.
 
 If given no arguments, will list the existing tags
 EOD
@@ -356,6 +378,8 @@ def tag(tag_name = nil, *packages)
                 desc: 'commit the package set state as well (enabled by default)'
             option :keep_going, aliases: :k, type: :boolean, banner: '',
                 desc: 'do not stop on build or checkout errors'
+            option :tag, aliases: :t, type: :string,
+                desc: 'the tag name to use'
             option :message, aliases: :m, type: :string,
                 desc: 'the message to use for the new commit (the default is to mention the creation of the tag)'
             def commit(*packages)
@@ -453,15 +477,36 @@ def manifest(*name)
             end
 
             desc 'exec', "runs a command, applying the workspace's environment first"
+            option :use_cache, type: :boolean, default: nil,
+                desc: "use the cached environment instead of "\
+                      "loading the whole configuration"
             def exec(*args)
                 require 'autoproj/cli/exec'
-                CLI::Exec.new.run(*args)
+                Autoproj.report(on_package_failures: default_report_on_package_failures, debug: options[:debug], silent: true) do
+                    opts = Hash.new
+                    use_cache = options[:use_cache]
+                    if !use_cache.nil?
+                        opts[:use_cached_env] = use_cache
+                    end
+                    CLI::Exec.new.run(*args, **opts)
+                end
             end
 
-            desc 'which', "resolves the full path to a command within the Autoproj workspace"
+            desc 'which', "resolves the full path to a command "\
+                " within the Autoproj workspace"
+            option :use_cache, type: :boolean, default: nil,
+                desc: "use the cached environment instead of "\
+                      "loading the whole configuration"
             def which(cmd)
                 require 'autoproj/cli/which'
-                CLI::Which.new.run(cmd)
+                Autoproj.report(on_package_failures: default_report_on_package_failures, debug: options[:debug], silent: true) do
+                    opts = Hash.new
+                    use_cache = options[:use_cache]
+                    if !use_cache.nil?
+                        opts[:use_cached_env] = use_cache
+                    end
+                    CLI::Which.new.run(cmd, **opts)
+                end
             end
         end
     end

data/lib/autoproj/cli/reset.rb

@@ -7,12 +7,13 @@ module Autoproj
     module CLI
         class Reset < InspectionTool
             def run(ref_name, options)
-                pkg = manifest.main_package_set.create_autobuild_package
+                ws.load_config
+                pkg = ws.manifest.main_package_set.create_autobuild_package
                 importer = pkg.importer
                 if !importer || !importer.kind_of?(Autobuild::Git)
                     raise CLIInvalidArguments, "cannot use autoproj reset if the main configuration is not managed by git"
                 end
-                
+
                 # Check if the reflog entry exists
                 begin
                     importer.rev_parse(pkg, ref_name)

data/lib/autoproj/cli/status.rb

@@ -90,6 +90,9 @@ def status_of_package(package_description, only_local: false, snapshot: false)
                             begin importer.snapshot(pkg, nil, exact_state: false, only_local: only_local)
                             rescue Autobuild::PackageException
                                 Hash.new
+                            rescue Exception => e
+                                package_status.msg << Autoproj.color("  failed to fetch snapshotting information (#{e})", :red)
+                                return package_status
                             end
                         if snapshot_overrides_vcs?(importer, package_description.vcs, snapshot_version)
                             non_nil_values = snapshot_version.delete_if { |k, v| !v }
@@ -198,7 +201,7 @@ def each_package_status(packages, parallel: ws.config.parallel_import_level, sna
                 Autoproj.warn "Interrupted, waiting for pending jobs to finish"
                 raise
             rescue Exception => e
-                Autoproj.error "internal error: #{e}, waiting for pending jobs to finish"
+                Autoproj.error "internal error (#{e.class}): #{e}, waiting for pending jobs to finish"
                 raise
             ensure
                 executor.shutdown

data/lib/autoproj/cli/versions.rb

@@ -9,7 +9,7 @@ class Versions < InspectionTool
             DEFAULT_VERSIONS_FILE_BASENAME = Ops::Snapshot::DEFAULT_VERSIONS_FILE_BASENAME
 
             def default_versions_file
-                File.join( Autoproj.overrides_dir, DEFAULT_VERSIONS_FILE_BASENAME )
+                File.join( ws.overrides_dir, DEFAULT_VERSIONS_FILE_BASENAME )
             end
 
             def validate_options(packages, options = Hash.new)
@@ -32,9 +32,9 @@ def run(user_selection, options)
                 packages, *, config_selected =
                     finalize_setup(user_selection,
                                    recursive: options[:deps])
-                
+
                 ops = Ops::Snapshot.new(ws.manifest, keep_going: options[:keep_going])
-                
+
                 if user_selection.empty?
                     snapshot_package_sets = (options[:config] != false)
                     snapshot_packages = !options[:config]

data/lib/autoproj/cli/watch.rb

@@ -0,0 +1,159 @@
+require 'rbconfig'
+require 'autoproj/cli/inspection_tool'
+require 'autoproj/ops/watch'
+module Autoproj
+    module CLI
+        class Watch < InspectionTool
+            attr_reader :notifier
+
+            def initialize(*args)
+                super(*args)
+                @show_events = false
+            end
+
+            def validate_options(unused, options = {})
+                _, options = super(unused, options)
+                @show_events   = options[:show_events]
+                nil
+            end
+
+            def show_events?
+                @show_events
+            end
+
+            def update_workspace
+                initialize_and_load
+
+                source_packages, _ = finalize_setup([])
+                @source_packages_dirs = source_packages.map do |pkg_name|
+                    ws.manifest.find_autobuild_package(pkg_name).srcdir
+                end
+                @pkg_sets_dirs = ws.manifest.each_package_set.map do |pkg_set|
+                    pkg_set.raw_local_dir
+                end
+                export_env_sh(shell_helpers: ws.config.shell_helpers?)
+            end
+
+            def load_info_from_installation_manifest
+                installation_manifest =
+                    begin
+                        Autoproj::InstallationManifest.from_workspace_root(ws.root_dir)
+                    rescue ConfigError
+                    end
+
+                @source_packages_dirs = []
+                @package_sets = []
+
+                @source_packages_dirs = installation_manifest.each_package.
+                    map(&:srcdir)
+                @package_sets = installation_manifest.each_package_set.
+                    map(&:raw_local_dir)
+            end
+
+            def callback
+                notifier.stop
+            end
+
+            def create_file_watcher(file)
+                notifier.watch(file, :modify) do |e|
+                    Autobuild.message "#{e.absolute_name} modified" if show_events?
+                    callback
+                end
+            end
+
+            def create_dir_watcher(dir, included_paths: [], excluded_paths: [], inotify_flags: [])
+                strip_dir_range = ((dir.size + 1)..-1)
+                notifier.watch(dir, :move, :create, :delete, :modify, :dont_follow, *inotify_flags) do |e|
+                    file_name = e.absolute_name[strip_dir_range]
+                    included = included_paths.empty? ||
+                        included_paths.any? { |rx| rx === file_name }
+                    if included
+                        included = !excluded_paths.any? { |rx| rx === file_name }
+                    end
+                    next if !included
+                    Autobuild.message "#{e.absolute_name} changed" if show_events?
+                    callback
+                end
+            end
+
+            def create_src_pkg_watchers
+                @source_packages_dirs.each do |pkg_srcdir|
+                    next unless File.exist? pkg_srcdir
+                    create_dir_watcher(pkg_srcdir, included_paths: ["manifest.xml"])
+
+                    manifest_file = File.join(pkg_srcdir, 'manifest.xml')
+                    next unless File.exist? manifest_file
+                    create_file_watcher(manifest_file)
+                end
+            end
+
+            def start_watchers
+                create_file_watcher(ws.config.path)
+                create_src_pkg_watchers
+                create_dir_watcher(ws.config_dir,
+                    excluded_paths: [/(^|#{File::SEPARATOR})\./],
+                    inotify_flags: [:recursive])
+                FileUtils.mkdir_p ws.remotes_dir
+                create_dir_watcher(ws.remotes_dir,
+                    excluded_paths: [/(^|#{File::SEPARATOR})\./],
+                    inotify_flags: [:recursive])
+            end
+
+            def cleanup_notifier
+                notifier.watchers.dup.each_value(&:close)
+                notifier.close
+            end
+
+            def assert_watchers_available
+                return if RbConfig::CONFIG['target_os'] =~ /linux/
+                puts 'error: Workspace watching not available on this platform'
+                exit 1
+            end
+
+            def setup_notifier
+                assert_watchers_available
+
+                require 'rb-inotify'
+                @notifier = INotify::Notifier.new
+            end
+
+            def cleanup
+                if @marker_io
+                    Ops.watch_cleanup_marker(@marker_io)
+                end
+                if @notifier
+                    cleanup_notifier
+                end
+            end
+
+            def restart
+                cleanup
+                args = []
+                args << "--show-events" if show_events?
+                exec($PROGRAM_NAME, 'watch', *args)
+            end
+
+            def run(**)
+                @marker_io = Ops.watch_create_marker(ws.root_dir)
+                begin
+                    update_workspace
+                rescue Exception => e
+                    puts "ERROR: #{e.message}"
+                    load_info_from_installation_manifest
+                end
+                setup_notifier
+                start_watchers
+
+                puts 'Watching workspace, press ^C to quit...'
+                notifier.run
+
+                puts 'Workspace changed...'
+                restart
+            rescue Interrupt
+                puts 'Exiting...'
+            ensure
+                cleanup
+            end
+        end
+    end
+end

data/lib/autoproj/cli/which.rb

@@ -1,14 +1,52 @@
-require 'autoproj/cli/inspection_tool'
+require 'autoproj'
+require 'autoproj/ops/cached_env'
+require 'autoproj/ops/which'
+require 'autoproj/ops/watch'
+
 module Autoproj
     module CLI
-        class Which < InspectionTool
-            def run(cmd)
-                initialize_and_load
-                finalize_setup(Array.new)
+        class Which
+            def initialize
+                @root_dir = Autoproj.find_workspace_dir
+                if !@root_dir
+                    require 'autoproj/workspace'
+                    # Will do all sorts of error reporting,
+                    # or may be able to resolve
+                    @root_dir = Workspace.default.root_dir
+                end
+            end
+
+            def load_cached_env
+                env = Ops.load_cached_env(@root_dir)
+                return if !env
+
+                Autobuild::Environment.
+                    environment_from_export(env, ENV)
+            end
+
+            def run(cmd, use_cached_env: Ops.watch_running?(@root_dir))
+                if use_cached_env
+                    env = load_cached_env
+                end
+
+                if !env
+                    require 'autoproj'
+                    require 'autoproj/cli/inspection_tool'
+                    ws = Workspace.from_dir(@root_dir)
+                    loader = InspectionTool.new(ws)
+                    loader.initialize_and_load
+                    loader.finalize_setup(Array.new)
+                    env = ws.full_env.resolved_env
+                end
 
-                puts ws.which(cmd)
-            rescue Workspace::ExecutableNotFound => e
+                path = env['PATH'].split(File::PATH_SEPARATOR)
+                puts Ops.which(cmd, path_entries: path)
+            rescue ExecutableNotFound => e
+                require 'autoproj' # make sure everything is available for error reporting
                 raise CLIInvalidArguments, e.message, e.backtrace
+            rescue Exception
+                require 'autoproj' # make sure everything is available for error reporting
+                raise
             end
         end
     end

data/lib/autoproj/configuration.rb

@@ -1,3 +1,9 @@
+require 'backports/2.4.0/float/dup'
+require 'backports/2.4.0/fixnum/dup'
+require 'backports/2.4.0/nil_class/dup'
+require 'backports/2.4.0/false_class/dup'
+require 'backports/2.4.0/true_class/dup'
+
 module Autoproj
     # Class that does the handling of configuration options as well as
     # loading/saving on disk
@@ -23,6 +29,18 @@ def initialize(path = nil)
             @declared_options = Hash.new
             @displayed_options = Hash.new
             @path = path
+            @modified = false
+        end
+
+        # Whether the configuration was changed since the last call to {#load}
+        # or {#save}
+        def modified?
+            @modified
+        end
+
+        # Resets the modified? flag to false
+        def reset_modified
+            @modified = false
         end
 
         # Deletes the current value for an option
@@ -32,7 +50,9 @@ def initialize(path = nil)
         # @param [String] the option name
         # @return the deleted value
         def reset(name)
+            @modified = config.has_key?(name)
             config.delete(name)
+            overrides.delete(name)
         end
 
         # Sets a configuration option
@@ -43,7 +63,12 @@ def reset(name)
         #   user about this value next time it is needed. Otherwise, it will be
         #   asked about it, the new value being used as default
         def set(key, value, user_validated = false)
-            config[key] = [value, user_validated]
+            if config.has_key?(key)
+                @modified = (config[key][0] != value)
+            else
+                @modified = true
+            end
+            config[key] = [value.dup, user_validated]
         end
 
         # Override a known option value
@@ -53,6 +78,16 @@ def override(option_name, value)
             overrides[option_name] = value
         end
 
+        # Remove a specific override
+        def reset_overrides(name)
+            @overrides.delete(name)
+        end
+
+        # Remove all overrides
+        def reset_overrides
+            @overrides.clear
+        end
+
         # Tests whether a value is set for the given option name
         #
         # @return [Boolean]
@@ -63,23 +98,31 @@ def has_value_for?(name)
         # Get the value for a given option
         def get(key, *default_value)
             if overrides.has_key?(key)
-                return overrides[key]
+                return overrides[key].dup
             end
 
+            has_value = config.has_key?(key)
             value, validated = config[key]
-            if value.nil? && !declared?(key) && !default_value.empty?
-                default_value.first
-            elsif value.nil? || (declared?(key) && !validated)
-                value = configure(key)
+
+            if !declared?(key)
+                if has_value
+                    return value.dup
+                elsif default_value.empty?
+                    raise ConfigError, "undeclared option '#{key}'"
+                else
+                    default_value.first.dup
+                end
             else
-                if declared?(key) && (displayed_options[key] != value)
+                if validated
                     doc = declared_options[key].short_doc
                     if doc[-1, 1] != "?"
                         doc = "#{doc}:"
                     end
                     displayed_options[key] = value
+                    value.dup
+                else
+                    configure(key).dup
                 end
-                value
             end
         end
 
@@ -135,6 +178,7 @@ def configure(option_name)
                     current_value = current_value.first
                 end
                 value = opt.ask(current_value)
+                @modified = true
                 config[option_name] = [value, true]
                 displayed_options[option_name] = value
                 value
@@ -143,14 +187,15 @@ def configure(option_name)
             end
         end
 
-        def load(options = Hash.new)
-            options = validate_options options,
-                path: self.path,
-                reconfigure: false
-
-            if h = YAML.load(File.read(options[:path]))
+        def load(path: self.path, reconfigure: false)
+            current_keys = @config.keys
+            if h = YAML.load(File.read(path))
                 h.each do |key, value|
-                    set(key, value, !options[:reconfigure])
+                    current_keys.delete(key)
+                    set(key, value, !reconfigure)
+                end
+                if current_keys.empty?
+                    @modified = false
                 end
             end
         end
@@ -160,11 +205,14 @@ def reconfigure!
             config.each do |key, (value, _user_validated)|
                 new_config[key] = [value, false]
             end
+            @modified = true
             @config = new_config
         end
 
-        def save(path = self.path)
-            File.open(path, "w") do |io|
+        def save(path = self.path, force: false)
+            return if !modified? && !force
+
+            Ops.atomic_write(path) do |io|
                 h = Hash.new
                 config.each do |key, value|
                     h[key] = value.first
@@ -172,6 +220,7 @@ def save(path = self.path)
 
                 io.write YAML.dump(h)
             end
+            @modified = false
         end
 
         def each_reused_autoproj_installation
@@ -474,6 +523,9 @@ def to_hash
             @config.each do |key, (value, _)|
                 result[key] = value
             end
+            overrides.each do |key, value|
+                result[key] = value
+            end
             result
         end
     end