restic-service 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5eb6ed2de697e518e95589565d8104e15aff50c4
+  data.tar.gz: afc4fd0c966455bb46920ce2ca5cd42902eb1e1c
+SHA512:
+  metadata.gz: 6c9e35b848ca8d1cb6627aefcc3c3073a3499b2770b995bf9d22ae673ff085f84193dbb52fa54a329ec6dd1dedf802dbe069ebcd8bd8f9ac932bd8806235ef9b
+  data.tar.gz: 31d8e7db6d33568552a33a90cef56c3c86996d6e1021a90f56b43224fda7a80e28a910465583a8c8186564784fb041abbb5c9302da66a2fbdda2c81d1ff380db

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.16.0

data/Gemfile

@@ -0,0 +1,11 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+group :development do
+    gem 'pry'
+    gem 'pry-byebug'
+end
+
+# Specify your gem's dependencies in restic-service.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Sylvain Joyeux
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,126 @@
+# restic-service
+
+A backup service using restic and rclone, meant to be run as a background
+service.
+
+## Usage
+
+`restic-service` expects a YAML configuration file. By default, it is stored in
+`/etc/restic-service/conf.yml`.
+
+The template configuration file looks like this:
+
+~~~ yaml
+# The path to the underlying tools
+tools:
+  restic: /opt/restic
+  rclone: /opt/rclone
+
+# The targets. The only generic parts of a target definition
+# are the name and type. The rest is target-specific
+targets:
+  - name: a_restic_sftp_target
+    type: restic-sftp
+~~~
+
+## Remote host identification using SSH {#ssh_key_id}
+
+Some target types recognize the remote host based on expected SSH keys.
+Corresponding keys have to be stored locally in the `keys/` subfolder of
+restic-service's configuration folder (i.e. /etc/restic-service/keys/ by
+default). Targets that are recognized this way must have a file named
+`${target_name}.keys`.
+
+This file can be created using ssh's tools with
+
+~~~
+ssh-keyscan -H hostname > /etc/restic-service/keys/targetname.keys
+~~~
+
+## Target type: restic-sftp
+
+The rest-sftp target backs files up using sftp on a remote target. The
+target expects the SFTP authentication to be done [using SSH keys](#ssh_key_id).
+
+The target takes the following arguments:
+
+~~~ yaml
+- name: name_of_target
+  type: restic-sftp
+  # The repo host. This is authenticated using SSH keys, so there must be a
+  # corresponding keys/name_of_target.keys file in $CONF_DIR/keys
+  host: host
+  # The username that should be used to connect to the host
+  username: the_user
+  # The repo path on the host
+  path: /
+  # The repository password (encryption password from Restic)
+  password: repo_password
+  # Mandatory, list of paths to backup. Needs at least one
+  includes:
+  - list
+  - of
+  - paths
+  - to
+  - backup
+  # Optional, list of excluded patterns
+  excludes:
+  - list/of/patterns
+  - to/not/backup
+  one_filesystem: false
+  # Optional, the IO class. Defaults to 3 (Idle)
+  io_class: 3
+  # Optional, the IO priority. Unused for IO class 3
+  io_priority: 0
+  # Optional, the CPU priority. Higher gets less CPU
+  cpu_priority: 19
+~~~
+
+## Target type: restic-b2
+
+The rest-b2 target backs files up using sftp on a B2 bucket.
+
+The target takes the following arguments:
+
+~~~ yaml
+- name: name_of_target
+  type: restic-b2
+  # The B2 bucket
+  bucket: mybucket
+  # The path within the bucket
+  path: path
+  # The B2 ID
+  id:
+  # The B2 Key
+  key:
+  # The repository password (encryption password from Restic)
+  password: repo_password
+  # Mandatory, list of paths to backup. Needs at least one
+  includes:
+  - list
+  - of
+  - paths
+  - to
+  - backup
+  # Optional, list of excluded patterns
+  excludes:
+  - list/of/patterns
+  - to/not/backup
+  one_filesystem: false
+  # Optional, the IO class. Defaults to 3 (Idle)
+  io_class: 3
+  # Optional, the IO priority. Unused for IO class 3
+  io_priority: 0
+  # Optional, the CPU priority. Higher gets less CPU
+  cpu_priority: 19
+~~~
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/ThirteenLtda/restic-service.
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "restic/service"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/restic-service

@@ -0,0 +1,9 @@
+#! /usr/bin/env ruby
+
+require 'restic/service/cli'
+
+begin
+    Restic::Service::CLI.start(ARGV)
+rescue Interrupt
+    STDERR.puts "Interrupted"
+end

data/install-dev.sh

@@ -0,0 +1,31 @@
+#! /bin/sh -ex
+
+PROGRAM=restic-service
+
+dev_path=$PWD
+
+target=`mktemp -d`
+cd $target
+cat > Gemfile <<GEMFILE
+source "https://rubygems.org"
+gem '${PROGRAM}', path: "$dev_path"
+GEMFILE
+
+cat Gemfile
+
+bundler install --standalone --binstubs
+if test -d /opt/${PROGRAM}; then
+    sudo rm -rf /opt/${PROGRAM}
+fi
+sudo cp -r . /opt/${PROGRAM}
+sudo chmod go+rX /opt/${PROGRAM}
+
+if test -d /lib/systemd/system; then
+    target_gem=`bundler show ${PROGRAM}`
+    sudo cp $target_gem/${PROGRAM}.service /lib/systemd/system
+    ( sudo systemctl stop ${PROGRAM}.service
+      sudo systemctl enable ${PROGRAM}.service
+      sudo systemctl start ${PROGRAM}.service )
+fi
+
+rm -rf $target

data/install.sh

@@ -0,0 +1,27 @@
+#! /bin/sh -ex
+
+PROGRAM=restic-service
+
+target=`mktemp -d`
+cd $target
+cat > Gemfile <<GEMFILE
+source "https://rubygems.org"
+gem '${PROGRAM}'
+GEMFILE
+
+bundler install --standalone --binstubs
+if test -d /opt/${PROGRAM}; then
+    sudo rm -rf /opt/${PROGRAM}
+fi
+sudo cp -r . /opt/${PROGRAM}
+sudo chmod go+rX /opt/${PROGRAM}
+
+if test -d /lib/systemd/system; then
+    target_gem=`bundler show ${PROGRAM}`
+    sudo cp $target_gem/${PROGRAM}.service /lib/systemd/system
+    ( sudo systemctl stop ${PROGRAM}.service
+      sudo systemctl enable ${PROGRAM}.service
+      sudo systemctl start ${PROGRAM}.service )
+fi
+
+rm -rf $target

data/lib/restic/service.rb

@@ -0,0 +1,14 @@
+require 'fileutils'
+require "pathname"
+require 'yaml'
+require 'tempfile'
+require "restic/service/version"
+require "restic/service/targets/base"
+require "restic/service/targets/restic"
+require "restic/service/targets/b2"
+require "restic/service/targets/restic_b2"
+require "restic/service/targets/restic_file"
+require "restic/service/targets/restic_sftp"
+require "restic/service/targets/rclone_b2"
+require "restic/service/ssh_keys"
+require "restic/service/conf"

data/lib/restic/service/cli.rb

@@ -0,0 +1,88 @@
+require 'thor'
+require 'restic/service'
+
+module Restic
+    module Service
+        DEFAULT_CONF = "/etc/restic-service"
+
+        class CLI < Thor
+            class_option :conf, desc: "path to the configuration file (#{DEFAULT_CONF})",
+                type: :string, default: DEFAULT_CONF
+
+            no_commands do
+                def conf_dir_path
+                    @conf_dir_path ||= Pathname.new(options[:conf])
+                end
+
+                def conf_file_path
+                    @conf_file_path ||= (conf_dir_path + "conf.yml")
+                end
+
+                def conf_keys_path
+                    @conf_keys_path ||= (conf_dir_path + "keys")
+                end
+
+                def load_conf
+                    Conf.load(conf_file_path)
+                end
+
+                def run_sync(conf, *targets)
+                    has_target = false
+                    conf.each_target do |target|
+                        has_target = true
+                        if !targets.empty? && !targets.include?(target.name)
+                            next
+                        end
+
+                        if !target.available?
+                            puts "#{target.name} is not available"
+                            next
+                        end
+
+                        puts
+                        puts "-----"
+                        puts "#{Time.now} - Synchronizing #{target.name}"
+                        target.run
+                    end
+                    if !has_target
+                        STDERR.puts "WARNING: no targets in #{options[:conf]}"
+                    end
+                end
+            end
+
+            desc 'available-targets', 'finds the available backup targets'
+            def whereami
+                STDOUT.sync = true
+                conf = load_conf
+                conf.each_target do |target|
+                    print "#{target.name}: "
+                    puts(target.available? ? 'yes' : 'no')
+                end
+            end
+
+            desc 'sync', 'synchronize all (some) targets'
+            def sync(*targets)
+                STDOUT.sync = true
+                conf = load_conf
+                run_sync(conf, *targets)
+            end
+
+            desc 'auto', 'periodically runs the backups, pass target names to restrict to these'
+            def auto(*targets)
+                STDOUT.sync = true
+                conf = load_conf
+                loop do
+                    puts "#{Time.now} Starting automatic synchronization pass"
+                    puts ""
+
+                    run_sync(conf, *targets)
+
+                    puts ""
+                    puts "#{Time.now} Finished automatic synchronization pass"
+
+                    sleep conf.period
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/conf.rb

@@ -0,0 +1,259 @@
+module Restic
+    module Service
+        # The overall service configuration
+        #
+        # This is the API side of the service configuration. The configuration
+        # is usually stored on disk in YAML and
+        #
+        # The YAML format is as follows:
+        #
+        #   # The path to the underlying tools
+        #   tools:
+        #     restic: /opt/restic
+        #     rclone: /opt/rclone
+        #
+        #   # The targets. The only generic parts of a target definition
+        #   # are the name and type. The rest is target-specific
+        #   targets:
+        #     - name: a_restic_sftp_target
+        #       type: restic-sftp
+        #
+        # See the README.md for more details about available targets
+        class Conf
+            # Exception raised when using a target name that does not exist
+            class NoSuchTarget < RuntimeError; end
+            # Exception raised when an invalid configuration file is loaded
+            class InvalidConfigurationFile < RuntimeError; end
+
+            # The default (empty) configuration
+            def self.default_conf
+                Hash['targets' => [],
+                     'period' => 3600,
+                     'bandwidth_limit' => nil,
+                     'tools' => Hash.new]
+            end
+
+            TARGET_CLASS_FROM_TYPE = Hash[
+                'restic-b2' => Targets::ResticB2,
+                'restic-sftp' => Targets::ResticSFTP,
+                'restic-file' => Targets::ResticFile,
+                'rclone-b2' => Targets::RcloneB2]
+
+            TOOLS = %w{restic rclone}
+
+            # Returns the target class that will handle the given target type
+            #
+            # @param [String] type the type as represented in the YAML file
+            def self.target_class_from_type(type)
+                if target_class = TARGET_CLASS_FROM_TYPE[type]
+                    return target_class
+                else
+                    raise InvalidConfigurationFile, "target type #{type} does not exist, available targets: #{TARGET_CLASS_FROM_TYPE.keys.sort.join(", ")}"
+                end
+            end
+
+            # Normalizes and validates a configuration hash, as stored in YAML
+            #
+            # @raise [InvalidConfigurationFile]
+            def self.normalize_yaml(yaml)
+                yaml = default_conf.merge(yaml)
+                TOOLS.each do |tool_name|
+                    yaml['tools'][tool_name] ||= tool_name
+                end
+
+                target_names = Array.new
+                yaml['targets'] = yaml['targets'].map do |target|
+                    if !target['name']
+                        raise InvalidConfigurationFile, "missing 'name' field in target"
+                    elsif !target['type']
+                        raise InvalidConfigurationFile, "missing 'type' field in target"
+                    end
+
+                    target_class = target_class_from_type(target['type'])
+                    if !target_class
+                        raise InvalidConfigurationFile, "target type #{target['type']} does not exist, available targets: #{TARGET_CLASS_FROM_TYPE.keys.sort.join(", ")}"
+                    end
+
+                    name = target['name'].to_s
+                    if target_names.include?(name)
+                        raise InvalidConfigurationFile, "duplicate target name '#{name}'"
+                    end
+
+                    target = target.dup
+                    target['name'] = name
+                    target = target_class.normalize_yaml(target)
+                    target_names << name
+                    target
+                end
+                yaml
+            end
+
+            # Load a configuration file
+            #
+            # @param [Pathname]
+            # @return [Conf]
+            # @raise (see normalize_yaml)
+            def self.load(path)
+                if !path.file?
+                    return Conf.new(Pathname.new(""))
+                end
+
+                yaml = YAML.load(path.read) || Hash.new
+                yaml = normalize_yaml(yaml)
+
+                conf = Conf.new(path.dirname)
+                conf.load_from_yaml(yaml)
+                conf
+            end
+
+            # The configuration path
+            #
+            # @return [Pathname]
+            attr_reader :conf_path
+
+            # The polling period in seconds
+            #
+            # Default is 1h (3600s)
+            #
+            # @return [Integer]
+            attr_reader :period
+
+            # The bandwidth limit in bytes/s
+            #
+            # Default is nil (none)
+            #
+            # @return [nil,Integer]
+            attr_reader :bandwidth_limit
+
+            def initialize(conf_path)
+                @conf_path = conf_path
+                @targets = Hash.new
+                @period  = 3600
+                @tools   = Hash.new
+                TOOLS.each do |tool_name|
+                    @tools[tool_name] = find_in_path(tool_name)
+                end
+            end
+
+            BANDWIDTH_SCALES = Hash[
+                nil => 1,
+                'k' => 1_000,
+                'm' => 1_000_000,
+                'g' => 1_000_000_000]
+
+            def self.parse_bandwidth_limit(limit)
+                if !limit.respond_to?(:to_str)
+                    return Integer(limit)
+                else
+                    match = /^(\d+)\s*(k|m|g)?$/.match(limit.downcase)
+                    if match
+                        return Integer(match[1]) * BANDWIDTH_SCALES.fetch(match[2])
+                    else
+                        raise ArgumentError, "cannot interpret '#{limit}' as a valid bandwidth limit, give a plain number in bytes or use the k, M and G suffixes"
+                    end
+                end
+            end
+
+            # The path to the key file for the given target
+            def conf_keys_path_for(target)
+                conf_path.join("keys", "#{target.name}.keys")
+            end
+
+            # Gets a target configuration
+            #
+            # @param [String] name the target name
+            # @return [Target]
+            # @raise NoSuchTarget
+            def target_by_name(name)
+                if target = @targets[name]
+                    target
+                else
+                    raise NoSuchTarget, "no target named '#{name}'"
+                end
+            end
+
+            # Enumerates the targets
+            def each_target(&block)
+                @targets.each_value(&block)
+            end
+
+            # Registers a target
+            def register_target(target)
+                @targets[target.name] = target
+            end
+
+            # @api private
+            #
+            # Helper that resolves a binary in PATH
+            def find_in_path(name)
+                ENV['PATH'].split(File::PATH_SEPARATOR).each do |p|
+                    candidate = Pathname.new(p).join(name)
+                    if candidate.file?
+                        return candidate
+                    end
+                end
+                nil
+            end
+
+            # Checks whether a given tool is available
+            #
+            # @param [String]
+            # @return [Boolean]
+            def tool_available?(tool_name)
+                @tools.has_key?(tool_name)
+            end
+
+            # The full path of a given tool
+            #
+            # @param [String]
+            # @return [Pathname]
+            def tool_path(tool_name)
+                if tool = @tools[tool_name]
+                    tool
+                else
+                    raise ArgumentError, "cound not find '#{tool_name}'"
+                end
+            end
+
+            # Add the information stored in a YAML-like hash into this
+            # configuration
+            #
+            # @param [Hash] the configuration, following the documented
+            #   configuration format (see {Conf})
+            # @return [void]
+            def load_from_yaml(yaml)
+                load_tools_from_yaml(yaml['tools'])
+                @period = Integer(yaml['period'])
+                @bandwidth_limit = if limit_yaml = yaml['bandwidth_limit']
+                                       Conf.parse_bandwidth_limit(limit_yaml)
+                                   end
+
+                yaml['targets'].each do |yaml_target|
+                    type = yaml_target['type']
+                    target_class = Conf.target_class_from_type(type)
+                    target = target_class.new(yaml_target['name'])
+                    target.setup_from_conf(self, yaml_target)
+                    register_target(target)
+                end
+            end
+
+            # @api private
+            #
+            # Helper for {#load_from_yaml}
+            def load_tools_from_yaml(yaml)
+                TOOLS.each do |tool_name|
+                    tool_path = Pathname.new(yaml[tool_name])
+                    if tool_path.relative?
+                        tool_path = find_in_path(tool_path)
+                    end
+                    if tool_path && tool_path.file?
+                        @tools[tool_name] = tool_path
+                    else
+                        STDERR.puts "cannot find path to #{tool_name}"
+                        @tools.delete(tool_name)
+                    end
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/ssh_keys.rb

@@ -0,0 +1,129 @@
+module Restic
+    module Service
+        # Interface to the functionality of querying and verifying host keys
+        class SSHKeys
+            class SSHFailed < RuntimeError; end
+            class NoLocalKey < RuntimeError; end
+            class ValidationFailed < RuntimeError; end
+
+            PublicKey = Struct.new :type, :hash do
+                def ==(other)
+                    other.type == type &&
+                        other.hash == hash
+                end
+            end
+            
+            # Load a set of SSH keys from a file
+            #
+            # @return [Array<PublicKey>]
+            def self.load_keys_from_file(path)
+                load_keys_from_string(path.read)
+            end
+
+            # Load a set of SSH keys from a string
+            #
+            # @return [Array<PublicKey>]
+            def self.load_keys_from_string(string)
+                string.each_line.map do |line|
+                    _, key_type, *rest = line.chomp.split(" ")
+                    PublicKey.new(key_type, rest.join(" "))
+                end
+            end
+
+            # Query the list of keys for this host
+            #
+            # @param [#host]
+            def query_keys(host)
+                self.class.load_keys_from_string(ssh_keyscan_host(host))
+            end
+
+            # Query the keys from a host and returns them as a SSH key string
+            def ssh_keyscan_host(host)
+                ssh_key_run('ssh-keyscan', '-H', host)
+            end
+
+            # Run a ssh subprocess and returns its standard output
+            #
+            # @raise [SSHFailed] if the subcommand fails
+            def ssh_key_run(*args)
+                out_pipe_r, out_pipe_w = IO.pipe
+                err_pipe_r, err_pipe_w = IO.pipe
+                pid = spawn *args, in: :close, err: err_pipe_w, out: out_pipe_w
+                out_pipe_w.close
+                err_pipe_w.close
+                _, status = Process.waitpid2 pid
+
+                out = out_pipe_r.read
+                err_pipe_r.readlines.each do |line|
+                    if line !~ /^#/
+                        STDERR.puts line
+                    end
+                end
+
+                if !status.success?
+t                    raise SSHFailed, "failed to run #{args}"
+                end
+                out
+            ensure
+                out_pipe_r.close
+                err_pipe_r.close
+            end
+
+            def ssh_config_path
+                Pathname.new(Dir.home) + ".ssh" + "config"
+            end
+
+            def ssh_setup_config(target_name, username, hostname, key_file, ssh_config_path: self.ssh_config_path)
+                ssh_config = ssh_cleanup_config(ssh_config_path: ssh_config_path)
+
+                if ssh_config[-1] && ssh_config[-1] != ''
+                    ssh_config << ""
+                end
+                ssh_config_name = "restic-service-host-#{target_name}"
+                ssh_config << "# Added by restic-service"
+                ssh_config << "Host #{ssh_config_name}"
+                ssh_config << "  User #{username}"
+                ssh_config << "  Hostname #{hostname}"
+                ssh_config << "  UserKnownHostsFile #{key_file}"
+
+                ssh_config_path.dirname.mkpath
+                ssh_config_path.dirname.chmod 0700
+                ssh_config_path.open('w') do |io|
+                    io.puts ssh_config.join("\n")
+                end
+                ssh_config_path.chmod 0600
+                ssh_config_name
+            end
+
+            def ssh_cleanup_config(ssh_config_path: self.ssh_config_path)
+                ssh_config =
+                    if ssh_config_path.file?
+                        ssh_config_path.read.split("\n").map(&:chomp)
+                    else
+                        []
+                    end
+
+                _, host_line = ssh_config.each_with_index.
+                    find { |line, line_i| line.start_with?("Host restic-service-host-") }
+                if host_line
+                    ssh_config.delete_at(host_line)
+                    while ssh_config[host_line - 1] && ssh_config[host_line - 1].start_with?("#")
+                        ssh_config.delete_at(host_line - 1)
+                        host_line -= 1
+                    end
+
+                    while ssh_config[host_line] && !ssh_config[host_line].start_with?("Host")
+                        ssh_config.delete_at(host_line)
+                    end
+                end
+                if ssh_config_path.file?
+                    ssh_config_path.open('w') do |io|
+                        io.puts ssh_config.join("\n")
+                    end
+                end
+                ssh_config
+            end
+        end
+    end
+end
+

data/lib/restic/service/targets/b2.rb

@@ -0,0 +1,38 @@
+module Restic
+    module Service
+        module Targets
+            module B2
+                def available?
+                    true
+                end
+
+                def self.normalize_yaml(yaml)
+                    %w{bucket path id key}.each do |required_field|
+                        if !yaml[required_field]
+                            raise Conf::InvalidConfigurationFile, "missing '#{required_field}' field in target"
+                        end
+                    end
+                    yaml
+                end
+
+                def initialize(*args)
+                    super
+
+                    @bucket = nil
+                    @path = nil
+                    @id = nil
+                    @key = nil
+                end
+
+                def setup_from_conf(conf, yaml)
+                    super
+
+                    @bucket = yaml['bucket']
+                    @path   = yaml['path']
+                    @id     = yaml['id']
+                    @key    = yaml['key']
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/targets/base.rb

@@ -0,0 +1,23 @@
+module Restic
+    module Service
+        module Targets
+            class Base
+                attr_reader :name
+
+                def initialize(name)
+                    @name = name
+
+                    @bandwidth_limit = nil
+                end
+
+                def setup_from_conf(conf, yaml)
+                    @bandwidth_limit =
+                        if limit = yaml.fetch('bandwidth_limit', conf.bandwidth_limit)
+                            Conf.parse_bandwidth_limit(limit)
+                        end
+                end
+            end
+        end
+    end
+end
+

data/lib/restic/service/targets/rclone_b2.rb

@@ -0,0 +1,49 @@
+module Restic
+    module Service
+        module Targets
+            class RcloneB2 < Base
+                include B2
+
+                def self.normalize_yaml(yaml)
+                    yaml = B2.normalize_yaml(yaml)
+                    if !yaml['src']
+                        raise Conf::InvalidConfigurationFile, "no src field provided for rclone-b2"
+                    elsif !File.directory?(yaml['src'])
+                        raise Conf::InvalidConfigurationFile, "provided rclone-b2 source #{yaml['src']} does not exist"
+                    end
+                    yaml
+                end
+
+                def setup_from_conf(conf, yaml)
+                    super
+                    @rclone_path = conf.tool_path('rclone')
+                    @src = yaml['src']
+                    @conf_path = conf.conf_path
+                end
+
+                def run
+                    extra_args = []
+                    if @bandwidth_limit
+                        extra_args << '--bwlimit' << @bandwidth_limit.to_s
+                    end
+
+                    Tempfile.create "rclone-#{@name}", @conf_path.to_path, perm: 0600 do |io|
+                        io.puts <<-EOCONF
+[restic-service]
+type = b2
+account = #{@id}
+key = #{@key}
+endpoint =
+EOCONF
+                        io.flush
+                        system(@rclone_path.to_path,
+                            '--transfers', '16',
+                            '--config', io.path,
+                            *extra_args,
+                            'sync', @src, "restic-service:#{@bucket}/#{@path}", in: :close)
+                    end
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/targets/restic.rb

@@ -0,0 +1,89 @@
+module Restic
+    module Service
+        module Targets
+            # Base class for all restic-based targets
+            #
+            # See README.md for the YAML configuration file format
+            class Restic < Base
+                def initialize(name)
+                    super
+
+                    @password = nil
+                    @includes = []
+                    @excludes = []
+                    @one_filesystem = false
+
+                    @io_class     = 3
+                    @io_priority  = 0
+                    @cpu_priority = 19
+                end
+
+                def one_filesystem?
+                    @one_filesystem
+                end
+
+                def self.normalize_yaml(yaml)
+                    yaml = Hash['includes' => [],
+                                'excludes' => [],
+                                'one_filesystem' => false,
+                                'io_class' => 3,
+                                'io_priority' => 0,
+                                'cpu_priority' => 19].merge(yaml)
+                    if yaml['includes'].empty?
+                        raise Conf::InvalidConfigurationFile, "nothing to backup"
+                    elsif !yaml['password']
+                        raise Conf::InvalidConfigurationFile, "no password field"
+                    end
+                    yaml
+                end
+
+                def setup_from_conf(conf, yaml)
+                    super
+
+                    @restic_path    = conf.tool_path('restic')
+                    @password       = yaml['password']
+                    @includes       = yaml['includes'] || Array.new
+                    @excludes       = yaml['excludes'] || Array.new
+                    @one_filesystem = !!yaml['one_filesystem']
+                    @io_class       = Integer(yaml['io_class'])
+                    @io_priority    = Integer(yaml['io_priority'])
+                    @cpu_priority   = Integer(yaml['cpu_priority'])
+                end
+
+                def run(*args)
+                    old_home = ENV['HOME']
+                    ENV['HOME'] = old_home || '/root'
+
+                    env = if args.first.kind_of?(Hash)
+                              env = args.shift
+                          else
+                              env = Hash.new
+                          end
+
+                    ionice_args = []
+                    if @io_class != 3
+                        ionice_args << '-n' << @io_priority.to_s
+                    end
+
+                    extra_args = []
+                    if one_filesystem?
+                        extra_args << '--one-file-system'
+                    end
+                    if @bandwidth_limit
+                        limit_KiB = @bandwidth_limit / 1000 
+                        extra_args << '--limit-download' << limit_KiB.to_s << '--limit-upload' << limit_KiB.to_s
+                    end
+
+                    system(Hash['RESTIC_PASSWORD' => @password].merge(env),
+                           'ionice', '-c', @io_class.to_s, *ionice_args,
+                           'nice', "-#{@cpu_priority}",
+                           @restic_path.to_path, *args, *extra_args,
+                           *@excludes.flat_map { |e| ['--exclude', e] },
+                           *@includes, in: :close)
+                ensure
+                    ENV['HOME'] = old_home
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/targets/restic_b2.rb

@@ -0,0 +1,21 @@
+module Restic
+    module Service
+        module Targets
+            # A target that backs up to a SFTP target using Restic
+            #
+            # See README.md for the YAML configuration file format
+            class ResticB2 < Restic
+                include B2
+
+                def self.normalize_yaml(yaml)
+                    yaml = B2.normalize_yaml(yaml)
+                    super(yaml)
+                end
+
+                def run
+                    super(Hash['B2_ACCOUNT_ID' => @id, 'B2_ACCOUNT_KEY' => @key], '-r', "b2:#{@bucket}:#{@path}", 'backup')
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/targets/restic_file.rb

@@ -0,0 +1,27 @@
+module Restic
+    module Service
+        module Targets
+            class ResticFile < Restic
+                def available?
+                    @dest.directory?
+                end
+
+                def self.normalize_yaml(yaml)
+                    if !yaml['dest']
+                        raise ArgumentError, "'dest' field not set in rest-file target"
+                    end
+                    super
+                end
+
+                def setup_from_conf(conf, target_yaml)
+                    super
+                    @dest = Pathname.new(target_yaml['dest'])
+                end
+
+                def run
+                    super('-r', @dest.to_path, 'backup')
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/targets/restic_sftp.rb

@@ -0,0 +1,61 @@
+module Restic
+    module Service
+        module Targets
+            # A target that backs up to a SFTP target using Restic
+            #
+            # See README.md for the YAML configuration file format
+            class ResticSFTP < Restic
+                def initialize(name)
+                    super
+                    @host = nil
+                    @username = nil
+                    @path = nil
+                    @host_keys = []
+                end
+
+                def available?
+                    ssh = SSHKeys.new
+                    actual_keys = ssh.query_keys(@host)
+                    valid?(actual_keys)
+                end
+
+                def self.normalize_yaml(yaml)
+                    %w{host username path password}.each do |required_field|
+                        if !yaml[required_field]
+                            raise Conf::InvalidConfigurationFile, "missing '#{required_field}' field in target"
+                        end
+                    end
+                    super
+                end
+
+                def setup_from_conf(conf, yaml)
+                    @target_name = yaml['name']
+                    @key_path    = conf.conf_keys_path_for(self)
+                    @host_keys = SSHKeys.load_keys_from_file(@key_path)
+                    @host      = yaml['host'].to_str
+                    @username  = yaml['username'].to_str
+                    @path      = yaml['path'].to_str
+                    @password  = yaml['password'].to_str
+                    super
+                end
+
+                def valid?(actual_keys)
+                    actual_keys.any? { |k| @host_keys.include?(k) }
+                end
+
+                def run
+                    current_home = ENV['HOME']
+                    ENV['HOME'] = current_home || '/root'
+
+                    ssh = SSHKeys.new
+                    ssh_config_name = ssh.ssh_setup_config(@target_name, @username, @host, @key_path)
+
+                    super('-r', "sftp:#{ssh_config_name}:#{@path}", 'backup')
+                ensure
+                    ssh.ssh_cleanup_config
+                    ENV['HOME'] = current_home
+                end
+            end
+        end
+    end
+end

data/lib/restic/service/version.rb

@@ -0,0 +1,5 @@
+module Restic
+  module Service
+    VERSION = "0.1.0"
+  end
+end

data/restic-service.gemspec

@@ -0,0 +1,29 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "restic/service/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "restic-service"
+  spec.version       = Restic::Service::VERSION
+  spec.authors       = ["Sylvain Joyeux"]
+  spec.email         = ["sylvain.joyeux@m4x.org"]
+
+  spec.summary       = %q{Higher-level management on top of restic to use it as a peridiodic backup tool}
+  spec.homepage      = "https://github.com/thirteenltda/restic-service"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "flexmock"
+end

data/restic-service.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=service that uses restic to periodically snapshot and backup
+
+[Service]
+Type=simple
+ExecStart=/opt/restic-service/bin/restic-service auto
+
+[Install]
+WantedBy=default.target

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: restic-service
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sylvain Joyeux
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-12-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- sylvain.joyeux@m4x.org
+executables:
+- restic-service
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/restic-service
+- install-dev.sh
+- install.sh
+- lib/restic/service.rb
+- lib/restic/service/cli.rb
+- lib/restic/service/conf.rb
+- lib/restic/service/ssh_keys.rb
+- lib/restic/service/targets/b2.rb
+- lib/restic/service/targets/base.rb
+- lib/restic/service/targets/rclone_b2.rb
+- lib/restic/service/targets/restic.rb
+- lib/restic/service/targets/restic_b2.rb
+- lib/restic/service/targets/restic_file.rb
+- lib/restic/service/targets/restic_sftp.rb
+- lib/restic/service/version.rb
+- restic-service.gemspec
+- restic-service.service
+homepage: https://github.com/thirteenltda/restic-service
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Higher-level management on top of restic to use it as a peridiodic backup
+  tool
+test_files: []