autoproj-sync 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: af756c3be787939a54e4d4f17cb1b8091737000678c8850e2e20741a1db0a5f2
+  data.tar.gz: 2640991f20d18f9972ddeeeab0aa77832987515b13a38829310983b157703c89
+SHA512:
+  metadata.gz: 0020ac4629195c4f9a802735cd1c3a5ec6f5e3fa3167d15d1efa1602a258422c300b8903c5228cc31574b7928c14b8329db56386d824355668848200f376f1fc
+  data.tar.gz: d0900fb01449b29bde7678b286932e72aa849f27b548e42ab930e2933d8cde636dde9893a612534ee52820875368ce346dce445bf9233c961c02d53b645ddc64

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+Gemfile.lock
+/vendor/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.16.2

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in autoproj-sync.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Sylvain Joyeux
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,112 @@
+# Autoproj::Sync
+
+This Autoproj plugin provides a way to keep remote location(s) synchronized
+with a local workspace, as part of the build process. This allows for to keep
+a local development workflow, but run things remotely in a very flexible way.
+
+It is not *that* magical. It is based on the assumption that:
+- the local and remote hosts have equivalent environments. In practice, it means
+  that binaries built with the local machine are compatible with the environment
+  on the remote machine (same shared libraries or using static libraries, ...)
+- the remote environment is synchronized at the exact same absolute path than
+  the local one
+
+## Installation
+
+Run
+
+```
+autoproj plugin install autoproj-sync
+```
+
+Autoproj Sync works only on workspaces that use separate prefixes. If you did
+not enable prefixes when bootstrapping your workspace, the easiest is to
+re-bootstrap a new clean one, passing the `--separate-prefixes` option.
+
+## Usage
+
+### Preparing the remote target
+
+To function, Autoproj Sync needs the remote target to match the local target, that is:
+
+- allow to run binaries built locally (meaning same or compatible shared libraries)
+- use the same full paths
+- have a compatible ruby binary at the same path than the one used locally. If you
+  for instance use rbenv, you will need to install the same Ruby version through rbenv.
+- if you are using a git version of Autoproj, or plugins that are not available through
+  RubyGems, you will have to install them manually at the same path than on the local
+  machine
+
+Moreover the remote target should be accessible via SSH public key authentication.
+
+### Automated Synchronisation
+
+Once a target is added and enabled (see below on how to do this), Autoproj will sync
+it automatically after an update or build operation. This works well with
+
+### Sync and the VSCode/Rock integration
+
+The vscode-rock extension can use sync to debug remote programs.
+
+This currently only supports C++ programs. When creating your launch entry, simply
+add the following line to it:
+
+~~~json
+"miDebuggerServerAddress": "rock:<remote_name>:<remote_port>"
+~~~
+
+where `<remote_name>` is the name of the Sync remote and `<remote_port>` the
+port that should be used by `gdbserver`. Run, and that's it.
+
+### CLI Usage
+
+Add and enable a synchronisation target with
+
+```
+autoproj sync add NAME URL
+```
+
+This will trigger a synchronization.
+
+Targets can be listed with
+
+```
+autoproj sync list
+```
+
+And removed with
+
+```
+autoproj sync remove NAME
+```
+
+Synchronisation can be temporarily disabled with
+
+```
+autoproj sync disable
+```
+
+And reenabled with
+
+```
+autoproj sync enable
+```
+
+Re-enabling a target will force a synchronisation to occur. Once a target is
+enabled, synchronisation happens during the build, whenever a package has
+been built and installed.
+
+The `enable` and `disable` subcommands both accept target names, which allows
+to selectively enable and disable.
+
+```
+autoproj sync enable
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rock-core/autoproj-sync.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+    t.libs << "test"
+    t.libs << "lib"
+    t.test_files = FileList["test/**/*_test.rb"]
+end

data/autoproj-sync.gemspec

@@ -0,0 +1,29 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "autoproj/sync/version"
+
+Gem::Specification.new do |spec|
+    spec.name          = "autoproj-sync"
+    spec.version       = Autoproj::Sync::VERSION
+    spec.authors       = ["Sylvain Joyeux"]
+    spec.email         = ["sylvain.joyeux@13robotics.com"]
+
+    spec.summary       = %q{Synchronizes the build byproducts of an Autoproj workspace to remote target(s)}
+    spec.homepage      = "https://rock-core.github.io/rock-and-syskit"
+    spec.license       = "MIT"
+
+    # Specify which files should be added to the gem when it is released.
+    # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+    spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+      `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    end
+    spec.bindir        = "exe"
+    spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+    spec.require_paths = ["lib"]
+
+    spec.add_dependency "autoproj", "~> 2.4"
+    spec.add_dependency "net-sftp"
+    spec.add_development_dependency "bundler", "~> 1.16"
+    spec.add_development_dependency "minitest", "~> 5.0"
+end

data/lib/autoproj-sync.rb

@@ -0,0 +1,39 @@
+require 'autoproj/cli/main_sync'
+
+class Autoproj::CLI::Main
+    desc 'sync', 'synchronize a workspace with remote location(s)'
+    subcommand 'sync', Autoproj::CLI::MainSync
+
+    register_post_command_hook(:build) do |ws, args|
+        source_packages = args[:source_packages]
+        source_packages = source_packages.map do |package_name|
+            ws.manifest.find_package_definition(package_name)
+        end
+
+        config = Autoproj::Sync::Config.new(ws)
+        config.each_enabled_remote.each do |remote|
+            remote.start do |sftp|
+                remote.update(sftp, ws, source_packages)
+            end
+        end
+    end
+    register_post_command_hook(:update) do |ws, args|
+        source_packages, osdep_packages = args.
+            values_at(:source_packages, :osdep_packages)
+        source_packages = source_packages.map do |package_name|
+            ws.manifest.find_package_definition(package_name)
+        end
+
+        config = Autoproj::Sync::Config.new(ws)
+        config.each_enabled_remote.each do |remote|
+            remote.start do |sftp|
+                unless source_packages.empty?
+                    remote.update(sftp, ws, source_packages)
+                end
+                unless osdep_packages.empty?
+                    remote.osdeps(sftp, ws, osdep_packages)
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/cli/main_sync.rb

@@ -0,0 +1,189 @@
+require 'autoproj/sync'
+
+module Autoproj
+    module CLI
+        # The 'jenkins' subcommand for autoproj
+        class MainSync < Thor
+            namespace 'sync'
+
+            no_commands do
+                def ws
+                    unless @ws
+                        @ws = Autoproj::Workspace.default
+                        @ws.load_config
+                        unless @ws.config.separate_prefixes?
+                            raise RuntimeError, "autoproj-sync only works on workspaces "\
+                                "that have separate prefixes enabled"
+                        end
+                    end
+                    @ws
+                end
+
+                def config
+                    unless @config
+                        @config = Sync::Config.new(ws)
+                    end
+                    @config
+                end
+
+                def ws_load
+                    ws.setup
+                    ws.load_package_sets
+                    ws.setup_all_package_directories
+                    ws.finalize_package_setup
+
+                    source_packages, osdep_packages, _resolved_selection =
+                        ws.load_packages(ws.manifest.default_packages(false),
+                            recursive: true,
+                            non_imported_packages: :ignore,
+                            auto_exclude: false)
+                    ws.finalize_setup
+                    source_packages = source_packages.map do |name|
+                        ws.manifest.find_package_definition(name)
+                    end
+                    [source_packages, osdep_packages]
+                end
+
+                def resolve_selected_remotes(*names)
+                    if names.empty?
+                        config.each_enabled_remote
+                    else
+                        names.map { |n| config.remote_by_name(n) }
+                    end
+                end
+
+            end
+
+            desc 'add NAME URL', "add a new remote target"
+            def add(name, uri)
+                if remote = config.find_remote_by_name(name)
+                    STDERR.puts "There is already a target called #{name} pointing to "\
+                        "#{remote.uri}"
+                    exit 1
+                end
+
+                remote = Sync::Remote.from_uri(URI.parse(uri), name: name)
+                config.add_remote(remote)
+                packages, = Autoproj.silent { ws_load }
+                remote.start do |sftp|
+                    remote.update(sftp, ws, packages)
+                end
+            end
+
+            desc 'remove NAME', "remove a remote target"
+            def remove(name)
+                config.delete_remote(name)
+            end
+
+            desc 'list', "lists registered targets"
+            def list
+                config.each_remote do |remote|
+                    enabled = remote.enabled? ? 'enabled' : 'disabled'
+                    puts "#{remote.name}: #{remote.uri} (#{enabled})"
+                end
+            end
+
+            desc 'status [NAME]', "accesses a target (or all enabled targets) and "\
+                "display outdated packages"
+            def status(*names)
+                remotes = resolve_selected_remotes(*names)
+                packages, = Autoproj.silent { ws_load }
+
+                remotes.each do |r|
+                    outdated_packages =
+                        r.start do |sftp|
+                            r.each_outdated_package(sftp, @ws, packages).to_a
+                        end
+                    puts "#{outdated_packages.size} outdated packages"
+                    outdated_packages.each do |pkg|
+                        puts "  #{pkg.name}"
+                    end
+                end
+            end
+
+            desc 'update NAME', "trigger an update for a remote"
+            def update(*names)
+                remotes = resolve_selected_remotes(*names)
+                packages, = Autoproj.silent { ws_load }
+
+                remotes.each do |r|
+                    r.start do |sftp|
+                        r.update(sftp, ws, packages)
+                    end
+                end
+            end
+
+            desc 'enable NAME', "enables a previously disabled target, or all targets"
+            def enable(*names)
+                names = ws.config.get('sync', Hash.new).keys if names.empty?
+                packages = nil
+                names.each do |name|
+                    config.update_remote_config(name) do |remote_config|
+                        unless remote_config['enabled']
+                            remote = config.remote_by_name(name)
+                            packages, = Autoproj.silent { ws_load } unless packages
+                            remote.start do |sftp|
+                                remote.update(sftp, ws, packages)
+                            end
+                            remote_config['enabled'] = true
+                        end
+                    end
+                end
+            end
+
+            desc 'disable [NAME]', "disables a previously enabled target, or all targets"
+            def disable(*names)
+                names = ws.config.get('sync', Hash.new).keys if names.empty?
+                names.each do |name|
+                    config.update_remote_config(name) do |config|
+                        config['enabled'] = false
+                    end
+                end
+            end
+
+            desc 'install-osdeps MANAGER_TYPE <PACKAGES...>',
+                'install osdeps coming from the local machine',
+                hide: true
+            def install_osdeps(manager_type, *packages)
+                Autobuild.silent { ws_load }
+                installer = ws.os_package_installer
+
+                installer.setup_package_managers
+                manager = installer.package_managers.fetch(manager_type)
+                installer.install_manager_packages(manager, packages)
+            end
+
+            desc 'osdeps [NAME]', 'install the osdeps on the remote'
+            def osdeps(*names)
+                _, osdep_packages = Autobuild.silent { ws_load }
+                remotes = resolve_selected_remotes(*names)
+                remotes.each do |r|
+                    r.start do |sftp|
+                        r.osdeps(sftp, ws, osdep_packages)
+                    end
+                end
+            end
+
+            desc 'exec NAME COMMAND', 'execute a command on a remote workspace'
+            option :interactive, doc: 'execute the command in interactive mode',
+                type: :boolean, default: false
+            option :chdir, doc: 'working directory for the command',
+                type: :string, default: nil
+            def exec(remote_name, *command)
+                remote = config.remote_by_name(remote_name)
+                status = remote.start do |sftp|
+                    remote.remote_autoproj(sftp, ws.root_dir, "exec", *command,
+                        interactive: options[:interactive],
+                        chdir: options[:chdir] || ws.root_dir)
+                end
+                unless options[:interactive]
+                    if status[:exit_signal]
+                        exit 255
+                    else
+                        exit status[:exit_code]
+                    end
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/sync.rb

@@ -0,0 +1,7 @@
+require 'autoproj'
+require 'uri'
+require 'pathname'
+require 'net/sftp'
+require 'autoproj/sync/version'
+require 'autoproj/sync/config'
+require 'autoproj/sync/remote'

data/lib/autoproj/sync/config.rb

@@ -0,0 +1,84 @@
+module Autoproj
+    module Sync
+        class Config
+            def initialize(ws)
+                @ws = ws
+            end
+
+            def each_remote
+                return enum_for(__method__) unless block_given?
+
+                targets = @ws.config.get('sync', Hash.new)
+                targets.each do |name, config|
+                    yield Remote.from_uri(URI.parse(config['uri']),
+                        name: name, enabled: config['enabled'])
+                end
+            end
+
+            def each_enabled_remote
+                return enum_for(__method__) unless block_given?
+
+                each_remote do |remote|
+                    yield(remote) if remote.enabled?
+                end
+            end
+
+            def remote_by_name(name)
+                unless remote = find_remote_by_name(name)
+                    raise ArgumentError, "no remote named '#{name}', "\
+                        "existing remotes: #{each_remote.map(&:name).sort.join(", ")}"
+                end
+                remote
+            end
+
+            def find_remote_by_name(name)
+                targets = @ws.config.get('sync', Hash.new)
+                if config = targets[name]
+                    remote_from_config(name, config)
+                end
+            end
+
+            private def remote_from_config(name, config)
+                Remote.from_uri(URI.parse(config['uri']),
+                    name: name, enabled: config['enabled'])
+            end
+
+            private def remote_to_config(remote)
+                [remote.name, Hash[
+                    'uri' => remote.uri.to_s,
+                    'enabled' => remote.enabled?
+                ]]
+            end
+
+            def add_remote(remote)
+                name, config = remote_to_config(remote)
+                targets = @ws.config.get('sync', Hash.new)
+                targets[name] = config
+                @ws.config.set('sync', targets)
+                @ws.save_config
+            end
+
+            def delete_remote(name)
+                targets = @ws.config.get('sync', Hash.new)
+                targets.delete(name)
+                @ws.config.set('sync', targets)
+                @ws.save_config
+            end
+
+            def update_remote_config(name)
+                targets = @ws.config.get('sync', Hash.new)
+                unless (config = targets[name])
+                    raise ArgumentError, "There is no target called #{name}"
+                end
+
+                new_config = config.dup
+                yield(new_config)
+                if new_config != config
+                    targets[name] = new_config
+                    @ws.config.set('sync', targets)
+                    @ws.save_config
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/sync/remote.rb

@@ -0,0 +1,399 @@
+# frozen_string_literal: true
+
+module Autoproj
+    module Sync
+        class Remote
+            class FailedRemoteCommand < RuntimeError; end
+
+            attr_reader :uri
+            attr_reader :name
+
+            def self.from_uri(uri, name: uri, enabled: true)
+                if uri.scheme != "ssh"
+                    raise ArgumentError, "unsupported protocol #{uri.scheme}"
+                end
+                Remote.new(uri, name: name, enabled: enabled)
+            end
+
+            def initialize(uri, name: uri, enabled: true)
+                @uri  = uri
+                @enabled = enabled
+                @name = name
+            end
+
+            def enabled?
+                @enabled
+            end
+
+            def remote_path
+                @uri.path
+            end
+
+            def start
+                result = nil
+                Net::SFTP.start(@uri.host, @uri.user, password: @uri.password) do |sftp|
+                    result = yield(sftp)
+                end
+                result
+            end
+
+            # Enumerate the packages that are outdated on the remote
+            #
+            # @yieldparam [Net::SFTP::Session] sftp the opened SFTP session, that can
+            #    be used to do further operations on the remote
+            # @yieldparam [Autoproj::PackageDescription] an outdated package
+            def each_outdated_package(sftp, ws, packages)
+                return enum_for(__method__, sftp, ws, packages) unless block_given?
+
+                stat = packages.map do |package|
+                    autobuild    = package.autobuild
+                    installstamp = autobuild.installstamp
+                    next unless File.exist?(installstamp)
+
+                    local_stat  = File.stat(installstamp)
+                    remote_path = File.join(uri.path, installstamp)
+                    begin
+                        remote_stat = sftp.file.open(remote_path) do |f|
+                            f.stat
+                        end
+                        [package, local_stat, remote_stat, remote_path]
+                    rescue Net::SFTP::StatusException => e
+                        if e.code != Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE
+                            raise
+                        end
+                        package
+                    end
+                end
+
+                stat.compact.map do |package, local_stat, remote_stat, remote_path|
+                    yield(package) if !local_stat ||
+                        changed_stat?(local_stat, remote_stat)
+                end.compact
+            end
+
+            private def changed_stat?(local, remote)
+                return true if local.size != remote.size
+
+                local_sec = local.mtime.tv_sec
+                local_usec = local.mtime.tv_usec
+                if remote.mtime != local_sec
+                    true
+                elsif !remote.respond_to?(:remote_nseconds)
+                    false
+                else
+                    (remote.mtime_nseconds / 1000) != local_usec
+                end
+            end
+
+            private def remote_mkdir_p(sftp, local_path)
+                remote_path = File.join(@uri.path, local_path)
+                ops = []
+                while remote_path != '/'
+                    ops << [remote_path, sftp.stat(remote_path)]
+                    remote_path = File.dirname(remote_path)
+                end
+                missing = ops.take_while do |_, op|
+                    op.wait
+                    !op.response.ok?
+                end
+                mkdirs = missing.reverse.map do |path, _|
+                    sftp.mkdir(path)
+                end
+                mkdirs.each(&:wait)
+            end
+
+            def autoproj_annex_files(ws)
+                user_files = %w[env.sh].
+                    map do |file|
+                        File.join(ws.root_dir, file)
+                    end
+                autoproj_files = %w[env.yml installation-manifest].
+                    map do |file|
+                        File.join(ws.root_dir, '.autoproj', file)
+                    end
+                bundler_files = %w[gems/Gemfile gems/Gemfile.lock].
+                    map do |file|
+                        File.join(ws.prefix_dir, file)
+                    end
+                [*user_files, *ws.env.source_before, *ws.env.source_after,
+                    *autoproj_files, *bundler_files]
+            end
+
+            def rsync_target
+                if @uri.user && @uri.password
+                    "#{@uri.user}:#{@uri.password}@#{@uri.host}"
+                elsif @uri.user
+                    "#{@uri.user}@#{@uri.host}"
+                else
+                    @uri.host
+                end
+            end
+
+            def rsync_dir(sftp, local_dir)
+                remote_dir = remote_path(local_dir)
+                ["rsync", "-a", "--delete-after", "#{local_dir}/",
+                    "#{rsync_target}:#{remote_dir}/"]
+            end
+
+            def rsync_file(sftp, local_file)
+                remote_file = remote_path(local_file)
+                ["rsync", "-a", local_file,
+                    "#{rsync_target}:#{remote_file}"]
+            end
+
+            def osdeps(sftp, ws, osdep_packages)
+                installer = ws.os_package_installer
+
+                installer.setup_package_managers
+                all = ws.all_os_packages
+                partitioned_packages = installer.
+                    resolve_and_partition_osdep_packages(osdep_packages, all)
+
+                os_packages = partitioned_packages.delete(installer.os_package_manager)
+                if os_packages
+                    partitioned_packages = [[installer.os_package_manager, os_packages]].
+                        concat(partitioned_packages.to_a)
+                end
+
+                partitioned_packages = partitioned_packages.map do |manager, packages|
+                    manager_name, _ = installer.package_managers.
+                        find { |key, obj| manager == obj }
+                    [manager_name, packages]
+                end
+
+                partitioned_packages.each do |manager_name, packages|
+                    install_osdep_packages(sftp, ws, manager_name, packages)
+                end
+            end
+
+            def install_osdep_packages(sftp, ws, manager_name, packages)
+                Autobuild.progress_start "sync-#{name}-osdeps-#{manager_name}",
+                    "sync: handling #{packages.size} osdeps #{manager_name} packages "\
+                        "on #{name}",
+                    done_message: "sync: handled #{packages.size} "\
+                        "osdeps #{manager_name} packages on #{name}" do
+
+                    result = remote_autoproj(sftp, ws.root_dir,
+                        "sync", "install-osdeps",
+                        manager_name, *packages)
+                    if result[:exit_code] != 0
+                        raise RuntimeError,
+                            "remote autoproj command failed\n"\
+                            "autoproj exited with status "\
+                            "#{result[:exit_code]}\n#{result}"
+                    end
+                end
+            end
+
+            def create_package_directories(sftp, pkg)
+                Autobuild.progress_start pkg, "sync: preparing #{pkg.name}@#{name}",
+                    done_message: "sync: prepared #{pkg.name}@#{name}" do
+
+                    remote_mkdir_p(sftp, pkg.autobuild.prefix)
+                    remote_mkdir_p(sftp, File.dirname(pkg.autobuild.installstamp))
+                end
+            end
+
+            def rsync_package(sftp, pkg)
+                Autobuild.progress_start pkg, "sync: updating #{pkg.name}@#{name}",
+                    done_message: "sync: updated #{pkg.name}@#{name}" do
+                    ops = [rsync_dir(sftp, pkg.autobuild.prefix),
+                        rsync_file(sftp, pkg.autobuild.installstamp)]
+                    ops.each do |op|
+                        if !system(*op)
+                            raise "update of #{pkg.name} failed"
+                        end
+                    end
+                end
+            end
+
+            def remote_path(local_path)
+                File.join(@uri.path, local_path)
+            end
+
+            def remote_file_exist?(sftp, path)
+                sftp.stat!(remote_path(path))
+                true
+            rescue Net::SFTP::StatusException => e
+                if e.code == Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE
+                    false
+                else
+                    raise
+                end
+            end
+
+            def remote_file_get(sftp, local_path)
+                sftp.download!(remote_path(local_path))
+            end
+
+            def remote_file_put(sftp, local_path, content)
+                sftp.upload!(StringIO.new(content), remote_path(local_path))
+            end
+
+            def remote_file_transfer(sftp, local_path, target: remote_path(local_path))
+                sftp.upload!(local_path, target)
+            end
+
+            def remote_autoproj(sftp, root_dir, *command, chdir: nil, interactive: false)
+                remote_exec(sftp,
+                    remote_path(File.join(root_dir, ".autoproj/bin/autoproj")),
+                    *command, chdir: chdir, interactive: interactive)
+            end
+
+            def remote_exec(sftp, *command, chdir: nil, interactive: false)
+                if interactive
+                    remote_interactive_exec(sftp, *command, chdir: chdir)
+                else
+                    status = Hash.new
+                    ios = Hash[:stdout => STDOUT, :stderr => STDOUT]
+                    target_dir = @uri.path
+                    target_dir = File.join(target_dir, chdir) if chdir
+                    pid = nil
+                    command = "cd '#{target_dir}' && "\
+                        "echo \"AUTOPROJ_SYNC_PID=$$\" && "\
+                        "exec '" + command.join("' '") + "'"
+                    ch = sftp.session.exec(command, status: status) do |channel, stream, data|
+                        if !pid && (m = /^AUTOPROJ_SYNC_PID=(\d+)/.match(data))
+                            pid = Integer(m[1])
+                        else
+                            ios[stream].print(data)
+                            ios[stream].flush
+                        end
+                    end
+
+                    begin
+                        ch.wait
+                        status
+                    rescue Interrupt
+                        sftp.session.exec!("kill #{pid}") if pid
+                        ch.wait
+                        raise
+                    end
+                end
+            end
+
+            def remote_interactive_exec(sftp, *command, chdir: nil)
+                channel = sftp.session.open_channel do |ch|
+                    ch.on_data do |ch, data|
+                        STDOUT.print data
+                        STDOUT.flush
+                    end
+                    ch.on_extended_data do |ch, type, data|
+                        STDERR.print data
+                    end
+
+                    ch.request_pty
+                    ch.exec("cd '#{chdir}' && '" + command.join("' '") + "'")
+                end
+
+                ssh = sftp.session
+                while channel.active?
+                    ssh.process(0.1)
+                    begin
+                        while true
+                            data = STDIN.read_nonblock(4096)
+                            channel.send_data(data)
+                        end
+                    rescue IO::WaitReadable
+                    end
+                end
+            rescue EOFError
+                channel.close
+            end
+
+            def local_file_get(local_path)
+                File.read(local_path)
+            end
+
+            def info(message)
+                Autoproj.message "  #{message}", force: true
+            end
+
+            def bootstrap_or_update_autoproj(sftp, ws)
+                gemfile_lock_path = File.join(ws.root_dir, ".autoproj/Gemfile.lock")
+                if remote_file_exist?(sftp, gemfile_lock_path)
+                    remote_gemfile_lock = remote_file_get(sftp, gemfile_lock_path)
+                    local_gemfile_lock  = local_file_get(gemfile_lock_path)
+                    if remote_gemfile_lock == local_gemfile_lock
+                        info "remote Autoproj install up-to-date"
+                        info "sync: Autoproj install up-to-date on #{name}"
+                        return
+                    end
+
+                    info "sync: updating the Autoproj install on #{name}"
+
+                    remote_file_put(sftp, gemfile_lock_path, local_gemfile_lock)
+                    remote_file_transfer(
+                        sftp, File.join(ws.root_dir, ".autoproj/Gemfile"))
+                    remote_file_transfer(
+                        sftp, File.join(ws.root_dir, ".autoproj/config.yml"))
+                    result = remote_autoproj(
+                        sftp, ws.root_dir, "update", "--autoproj")
+                    unless result[:exit_code] == 0
+                        raise FailedRemoteCommand, "failed to update Autoproj:\n"\
+                            "autoproj update --autoproj finished with exit status "\
+                            "#{result[:exit_code]}\n"\
+                            "#{result}"
+                    end
+                else
+                    info "sync: installing Autoproj on #{name}"
+
+                    autoproj_spec = Bundler.definition.specs.
+                        find { |spec| spec.name == "autoproj" }
+                    autoproj_dir = autoproj_spec.full_gem_path
+                    install_script = File.join(autoproj_dir, "bin", "autoproj_install")
+                    remote_mkdir_p(sftp, ws.root_dir)
+                    sftp.upload!(install_script,
+                        remote_path(File.join(ws.root_dir, "autoproj_install")))
+                    remote_file_transfer(sftp, File.join(ws.root_dir, ".autoproj/config.yml"),
+                        target: remote_path(File.join(ws.root_dir, 'bootstrap-config.yml')))
+                    remote_file_transfer(sftp, File.join(ws.root_dir, ".autoproj/Gemfile"),
+                        target: remote_path(File.join(ws.root_dir, 'bootstrap-Gemfile')))
+                    result = sftp.session.exec!("cd '#{remote_path(ws.root_dir)}' && "\
+                        "#{ws.config.ruby_executable} autoproj_install "\
+                        "--gemfile bootstrap-Gemfile "\
+                        "--seed-config bootstrap-config.yml")
+                    if result.exitstatus != 0
+                        raise RuntimeError, "failed to install autoproj: #{result}"
+                    end
+                end
+            end
+
+            def update(sftp, ws, packages)
+                # First check if autoproj is bootstrapped on the target already
+                bootstrap_or_update_autoproj(sftp, ws)
+
+                packages = each_outdated_package(sftp, ws, packages).to_a
+
+                info "sync: #{packages.size} outdated packages on #{name}"
+
+                executor = Concurrent::FixedThreadPool.new(6)
+                futures = packages.map do |pkg|
+                    create_package_directories(sftp, pkg)
+                    Concurrent::Future.execute(executor: executor) do
+                        rsync_package(sftp, pkg)
+                    end
+                end
+
+                # Copy some autoproj installation-manifest files
+                Autobuild.progress_start "sync-#{name}-autoproj",
+                    "sync: updating Autoproj configuration files on #{name}",
+                    done_message: "sync: updated Autoproj configuration files on #{name}" do
+                    autoproj_annex_files(ws).each do |file|
+                        sftp.upload!(file, File.join(@uri.path, file))
+                    end
+                end
+
+                futures.each_with_index do |f, i|
+                    f.value!
+                end
+
+            ensure
+                if executor
+                    executor.shutdown
+                    executor.wait_for_termination
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/sync/version.rb

@@ -0,0 +1,5 @@
+module Autoproj
+    module Sync
+        VERSION = "0.1.0"
+    end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: autoproj-sync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sylvain Joyeux
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: autoproj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: 
+email:
+- sylvain.joyeux@13robotics.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- autoproj-sync.gemspec
+- lib/autoproj-sync.rb
+- lib/autoproj/cli/main_sync.rb
+- lib/autoproj/sync.rb
+- lib/autoproj/sync/config.rb
+- lib/autoproj/sync/remote.rb
+- lib/autoproj/sync/version.rb
+homepage: https://rock-core.github.io/rock-and-syskit
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Synchronizes the build byproducts of an Autoproj workspace to remote target(s)
+test_files: []