restic-service 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5eb6ed2de697e518e95589565d8104e15aff50c4
-  data.tar.gz: afc4fd0c966455bb46920ce2ca5cd42902eb1e1c
+  metadata.gz: 4f4ae91f2cc1e092886efb337eaf4fcc6b31f1e8
+  data.tar.gz: c1aed6a2ed9345f1209c36b4cfb19e537607a1d4
 SHA512:
-  metadata.gz: 6c9e35b848ca8d1cb6627aefcc3c3073a3499b2770b995bf9d22ae673ff085f84193dbb52fa54a329ec6dd1dedf802dbe069ebcd8bd8f9ac932bd8806235ef9b
-  data.tar.gz: 31d8e7db6d33568552a33a90cef56c3c86996d6e1021a90f56b43224fda7a80e28a910465583a8c8186564784fb041abbb5c9302da66a2fbdda2c81d1ff380db
+  metadata.gz: 968968cee27de96246836215811f1d50ce80f6039b33f5c9b66539c60843c555c6e2601b09d4b95a161e47d43029835263c2cf5a01f77ceaa110bb7e7bcfb465
+  data.tar.gz: abc0903185556bf615d1f4ec1c5c1632f716b74757647e857189d8637505ae397fdeb04528e61dee2a86ac31185db78f56afb4f73044b099f9c2b73d41710579

data/install-dev.sh

@@ -4,6 +4,10 @@ PROGRAM=restic-service
 
 dev_path=$PWD
 
+if test "x$1" = "x--systemd"; then
+    systemd=1
+fi
+
 target=`mktemp -d`
 cd $target
 cat > Gemfile <<GEMFILE
@@ -11,16 +15,22 @@ source "https://rubygems.org"
 gem '${PROGRAM}', path: "$dev_path"
 GEMFILE
 
-cat Gemfile
+bundler install --binstubs --without development --path vendor
+for i in vendor/ruby/*; do
+    gem_home_relative=$i
+done
+GEM_HOME=$PWD/$gem_home_relative gem install bundler --no-document
+for stub in bin/*; do
+    sed -i "/usr.bin.env/a ENV['GEM_HOME']='/opt/restic-service/$gem_home_relative'" $stub
+done
 
-bundler install --standalone --binstubs
 if test -d /opt/${PROGRAM}; then
     sudo rm -rf /opt/${PROGRAM}
 fi
 sudo cp -r . /opt/${PROGRAM}
 sudo chmod go+rX /opt/${PROGRAM}
 
-if test -d /lib/systemd/system; then
+if test "x$systemd" = "x1" && test -d /lib/systemd/system; then
     target_gem=`bundler show ${PROGRAM}`
     sudo cp $target_gem/${PROGRAM}.service /lib/systemd/system
     ( sudo systemctl stop ${PROGRAM}.service

data/install.sh

@@ -2,6 +2,10 @@
 
 PROGRAM=restic-service
 
+if test "x$1" = "x--systemd"; then
+    systemd=1
+fi
+
 target=`mktemp -d`
 cd $target
 cat > Gemfile <<GEMFILE
@@ -9,14 +13,22 @@ source "https://rubygems.org"
 gem '${PROGRAM}'
 GEMFILE
 
-bundler install --standalone --binstubs
+bundler install --binstubs --without development --path vendor
+for i in vendor/ruby/*; do
+    gem_home_relative=$i
+done
+GEM_HOME=$PWD/$gem_home_relative gem install bundler --no-document
+for stub in bin/*; do
+    sed -i "/usr.bin.env/a ENV['GEM_HOME']='/opt/restic-service/$gem_home_relative'" $stub
+done
+
 if test -d /opt/${PROGRAM}; then
     sudo rm -rf /opt/${PROGRAM}
 fi
 sudo cp -r . /opt/${PROGRAM}
 sudo chmod go+rX /opt/${PROGRAM}
 
-if test -d /lib/systemd/system; then
+if test "x$systemd" = "x1" && test -d /lib/systemd/system; then
     target_gem=`bundler show ${PROGRAM}`
     sudo cp $target_gem/${PROGRAM}.service /lib/systemd/system
     ( sudo systemctl stop ${PROGRAM}.service

data/lib/restic/service.rb

@@ -3,6 +3,7 @@ require "pathname"
 require 'yaml'
 require 'tempfile'
 require "restic/service/version"
+require "restic/service/auto_update"
 require "restic/service/targets/base"
 require "restic/service/targets/restic"
 require "restic/service/targets/b2"

data/lib/restic/service/auto_update.rb

@@ -0,0 +1,106 @@
+require 'net/https'
+
+module Restic
+    module Service
+        class AutoUpdate
+            class FailedUpdate < RuntimeError
+            end
+
+            RESTIC_RELEASE_VERSION = "0.8.3"
+
+            def self.restic_release_url(platform)
+                "https://github.com/restic/restic/releases/download/v#{RESTIC_RELEASE_VERSION}/restic_#{RESTIC_RELEASE_VERSION}_#{platform}.bz2"
+            end
+
+
+            def initialize(binary_path)
+                @root = File.dirname(File.dirname(binary_path))
+                unless File.file?(File.join(@root, "Gemfile"))
+                    raise FailedUpdate, "cannot guess installation path (tried #{@root})"
+                end
+
+                @gem_home = ENV['GEM_HOME']
+            end
+
+            def patch_binstubs
+                bindir = File.join(@root, 'bin')
+                Dir.new(bindir).each do |entry|
+                    entry = File.join(bindir, entry)
+                    if File.file?(entry)
+                        patched_contents = File.readlines(entry)
+                        patched_contents.insert(1, "ENV['GEM_HOME'] = '#{@gem_home}'\n")
+                        File.open(entry, 'w') do |io|
+                            io.write patched_contents.join("")
+                        end
+                    end
+                end
+            end
+
+            def current_gem_version(gem_name)
+                gemfile_lock = File.join(@root, "Gemfile.lock")
+                File.readlines(gemfile_lock).each do |line|
+                    match = /^\s+#{gem_name} \((.*)\)$/.match(line)
+                    return match[1] if match
+                end
+                raise FailedUpdate, "cannot find the version line for #{gem_name} in #{gemfile_lock}"
+            end
+
+            def update_restic_service
+                current_version = current_gem_version "restic-service"
+                reader, writer = IO.pipe
+                if !system("bundle", "update", out: writer, err: writer)
+                    writer.close
+                    puts reader.read(1024)
+                    raise FailedUpdate, "failed to run bundle update"
+                end
+                patch_binstubs
+                new_version = current_gem_version "restic-service"
+                [current_version, new_version]
+            ensure
+                writer.close if writer && !writer.closed?
+                reader.close if reader && !reader.closed?
+            end
+
+            def update_restic(platform, target_path)
+                release_url = self.class.restic_release_url(platform)
+
+                release_binary = nil
+                while !release_binary
+                    response = Net::HTTP.get_response(URI(release_url))
+                    case response
+                    when Net::HTTPSuccess
+                        release_binary = response.body
+                    when Net::HTTPRedirection
+                        release_url = response['location']
+                    else
+                        raise FailedUpdate, "failed to fetch restic at #{release_url}: #{response}"
+                    end
+                end
+
+                tmpdir = Dir.mktmpdir
+                restic_path = File.join(tmpdir, "restic")
+                File.open("#{restic_path}.bz2", 'w') do |io|
+                    io.write release_binary
+                end
+
+                if !system("bzip2", "-d", "#{restic_path}.bz2")
+                    raise FailedUpdate, "failed to uncompress the restic release file"
+                end
+
+                if File.file?(target_path)
+                    current = File.read(target_path)
+                    new     = File.read(restic_path)
+                    return if current == new
+                end
+
+                FileUtils.mv restic_path, target_path
+                FileUtils.chmod 0755, target_path
+                true
+
+            ensure
+                FileUtils.rm_rf tmpdir if tmpdir
+            end
+        end
+    end
+end
+

data/lib/restic/service/cli.rb

@@ -26,31 +26,50 @@ module Restic
                     Conf.load(conf_file_path)
                 end
 
-                def run_sync(conf, *targets)
+                def each_selected_and_available_target(conf, *targets)
                     has_target = false
                     conf.each_target do |target|
                         has_target = true
                         if !targets.empty? && !targets.include?(target.name)
                             next
-                        end
-
-                        if !target.available?
+                        elsif !target.available?
                             puts "#{target.name} is not available"
                             next
                         end
 
+                        yield(target)
+                    end
+
+                    if !has_target
+                        STDERR.puts "WARNING: no targets in #{options[:conf]}"
+                    end
+                end
+
+                def run_sync(conf, *targets)
+                    each_selected_and_available_target(conf, *targets) do |target|
                         puts
                         puts "-----"
                         puts "#{Time.now} - Synchronizing #{target.name}"
                         target.run
                     end
-                    if !has_target
-                        STDERR.puts "WARNING: no targets in #{options[:conf]}"
+                end
+
+                def run_forget(conf, *targets)
+                    each_selected_and_available_target(conf, *targets) do |target|
+                        unless target.respond_to?(:forget)
+                            puts "#{target.name} does not supports forget"
+                            next
+                        end
+
+                        puts
+                        puts "-----"
+                        puts "#{Time.now} - Running forget pass on #{target.name}"
+                        target.forget
                     end
                 end
             end
 
-            desc 'available-targets', 'finds the available backup targets'
+            desc 'whereami', 'finds the available backup targets'
             def whereami
                 STDOUT.sync = true
                 conf = load_conf
@@ -60,6 +79,53 @@ module Restic
                 end
             end
 
+            desc 'install-restic PATH PLATFORM', 'install restic'
+            def install_restic(path, platform)
+                updater = AutoUpdate.new($0)
+                updater.update_restic(platform, path)
+            end
+
+            desc 'auto-update', 'perform auto-updating as configured in the configuration file'
+            def auto_update
+                conf = load_conf
+                STDOUT.sync = true
+
+                updater = AutoUpdate.new($0)
+                if conf.auto_update_restic_service?
+                    puts "attempting to auto-update restic-service"
+                    old_version, new_version = updater.update_restic_service
+                    if old_version != new_version
+                        puts "updated restic-service from #{old_version} to #{new_version}, restarting"
+                        exec "bundle", "exec", Gem.ruby, $0, "auto-update"
+                    else
+                        puts "restic-service was already up-to-date: #{new_version}"
+                    end
+                else
+                    puts "updating restic-service disabled in configuration"
+                end
+
+                update_restic = conf.auto_update_restic?
+                if update_restic
+                    begin
+                        restic_path = conf.tool_path('restic')
+                    rescue ArgumentError
+                        puts "cannot auto-update restic, provide an explicit path in the 'tools' section of the configuration first"
+                        update_restic = false
+                    end
+                else
+                    puts "updating restic disabled in configuration"
+                end
+
+                if update_restic
+                    puts "attempting to auto-update restic"
+                    if updater.update_restic(conf.restic_platform, restic_path)
+                        puts "updated restic to version #{AutoUpdate::RESTIC_RELEASE_VERSION}"
+                    else
+                        puts "restic was already up-to-date"
+                    end
+                end
+            end
+
             desc 'sync', 'synchronize all (some) targets'
             def sync(*targets)
                 STDOUT.sync = true
@@ -67,6 +133,13 @@ module Restic
                 run_sync(conf, *targets)
             end
 
+            desc 'forget', 'delete historical data'
+            def forget(*targets)
+                STDOUT.sync = true
+                conf = load_conf
+                run_forget(conf, *targets)
+            end
+
             desc 'auto', 'periodically runs the backups, pass target names to restrict to these'
             def auto(*targets)
                 STDOUT.sync = true
@@ -76,6 +149,7 @@ module Restic
                     puts ""
 
                     run_sync(conf, *targets)
+                    run_forget(conf, *targets)
 
                     puts ""
                     puts "#{Time.now} Finished automatic synchronization pass"

data/lib/restic/service/conf.rb

@@ -61,6 +61,8 @@ module Restic
                     yaml['tools'][tool_name] ||= tool_name
                 end
 
+                yaml['auto_update'] ||= Array.new
+
                 target_names = Array.new
                 yaml['targets'] = yaml['targets'].map do |target|
                     if !target['name']
@@ -215,6 +217,18 @@ module Restic
                 end
             end
 
+            def auto_update_restic_service?
+                @auto_update_restic_service
+            end
+
+            def auto_update_restic?
+                @auto_update_restic
+            end
+
+            def restic_platform
+                @auto_update_restic
+            end
+
             # Add the information stored in a YAML-like hash into this
             # configuration
             #
@@ -228,6 +242,14 @@ module Restic
                                        Conf.parse_bandwidth_limit(limit_yaml)
                                    end
 
+                yaml['auto_update'].each do |update_target, do_update|
+                    if update_target == 'restic-service'
+                        @auto_update_restic_service = do_update
+                    elsif update_target == 'restic'
+                        @auto_update_restic = do_update
+                    end
+                end
+
                 yaml['targets'].each do |yaml_target|
                     type = yaml_target['type']
                     target_class = Conf.target_class_from_type(type)

data/lib/restic/service/targets/restic.rb

@@ -48,40 +48,81 @@ module Restic
                     @io_class       = Integer(yaml['io_class'])
                     @io_priority    = Integer(yaml['io_priority'])
                     @cpu_priority   = Integer(yaml['cpu_priority'])
+                    @forget         = parse_forget_setup(yaml['forget'] || Hash.new)
                 end
 
-                def run(*args)
-                    old_home = ENV['HOME']
-                    ENV['HOME'] = old_home || '/root'
+                Forget = Struct.new :prune, :tags, :hourly, :daily, :weekly, :monthly, :yearly do
+                    def prune?
+                        prune
+                    end
+                end
+                FORGET_DURATION_KEYS = %w{tags hourly daily weekly monthly yearly}
+                FORGET_KEYS = ['prune', *FORGET_DURATION_KEYS].freeze
+                def parse_forget_setup(setup)
+                    parsed = Forget.new true, []
+                    if (invalid_key = setup.find { |k, _| !FORGET_KEYS.include?(k) })
+                        raise ArgumentError, "#{invalid_key} is not a valid key within "\
+                            "'forget', valid keys are: #{FORGET_KEYS.join(", ")}"
+                    end
 
+                    FORGET_KEYS.each do |key|
+                        parsed[key] = setup.fetch(key, key == "prune")
+                    end
+                    parsed
+                end
+
+                def run_backup(*args, **options)
+                    extra_args = []
+                    if one_filesystem?
+                        extra_args << '--one-file-system'
+                    end
+
+                    run_restic(*args, *extra_args,
+                           *@excludes.flat_map { |e| ['--exclude', e] },
+                           *@includes)
+                end
+
+                def run_restic(*args, **options)
+                    home = ENV['HOME'] || '/root'
                     env = if args.first.kind_of?(Hash)
                               env = args.shift
                           else
                               env = Hash.new
                           end
 
-                    ionice_args = []
-                    if @io_class != 3
-                        ionice_args << '-n' << @io_priority.to_s
-                    end
-
                     extra_args = []
-                    if one_filesystem?
-                        extra_args << '--one-file-system'
-                    end
                     if @bandwidth_limit
                         limit_KiB = @bandwidth_limit / 1000 
                         extra_args << '--limit-download' << limit_KiB.to_s << '--limit-upload' << limit_KiB.to_s
                     end
 
-                    system(Hash['RESTIC_PASSWORD' => @password].merge(env),
+                    ionice_args = []
+                    if @io_class != 3
+                        ionice_args << '-n' << @io_priority.to_s
+                    end
+
+                    system(Hash['HOME' => home, 'RESTIC_PASSWORD' => @password].merge(env),
                            'ionice', '-c', @io_class.to_s, *ionice_args,
                            'nice', "-#{@cpu_priority}",
-                           @restic_path.to_path, *args, *extra_args,
-                           *@excludes.flat_map { |e| ['--exclude', e] },
-                           *@includes, in: :close)
-                ensure
-                    ENV['HOME'] = old_home
+                           @restic_path.to_path, "--cleanup-cache", *args, *extra_args, in: :close, **options)
+                end
+
+                def run_forget(*args)
+                    extra_args = []
+                    FORGET_DURATION_KEYS.each do |key|
+                        arg_key =
+                            if key == "tags" then "tag"
+                            else key
+                            end
+                        if value = @forget[key]
+                            extra_args << "--keep-#{arg_key}" << value.to_s
+                        end
+                    end
+                    if @forget.prune?
+                        extra_args << "--prune"
+                        puts "PRUNE"
+                    end
+                    run_restic(*args, *extra_args)
                 end
             end
         end

data/lib/restic/service/targets/restic_b2.rb

@@ -13,7 +13,13 @@ module Restic
                 end
 
                 def run
-                    super(Hash['B2_ACCOUNT_ID' => @id, 'B2_ACCOUNT_KEY' => @key], '-r', "b2:#{@bucket}:#{@path}", 'backup')
+                    run_backup(Hash['B2_ACCOUNT_ID' => @id, 'B2_ACCOUNT_KEY' => @key],
+                          '-r', "b2:#{@bucket}:#{@path}", 'backup')
+                end
+
+                def forget
+                    run_forget(Hash['B2_ACCOUNT_ID' => @id, 'B2_ACCOUNT_KEY' => @key],
+                          '-r', "b2:#{@bucket}:#{@path}", 'forget')
                 end
             end
         end

data/lib/restic/service/targets/restic_sftp.rb

@@ -44,16 +44,21 @@ module Restic
                 end
 
                 def run
-                    current_home = ENV['HOME']
-                    ENV['HOME'] = current_home || '/root'
+                    ssh = SSHKeys.new
+                    ssh_config_name = ssh.ssh_setup_config(@target_name, @username, @host, @key_path)
+
+                    run_backup('-r', "sftp:#{ssh_config_name}:#{@path}", 'backup')
+                ensure
+                    ssh.ssh_cleanup_config
+                end
 
+                def forget
                     ssh = SSHKeys.new
                     ssh_config_name = ssh.ssh_setup_config(@target_name, @username, @host, @key_path)
 
-                    super('-r', "sftp:#{ssh_config_name}:#{@path}", 'backup')
+                    run_forget('-r', "sftp:#{ssh_config_name}:#{@path}", 'forget')
                 ensure
                     ssh.ssh_cleanup_config
-                    ENV['HOME'] = current_home
                 end
             end
         end

data/lib/restic/service/version.rb

@@ -1,5 +1,5 @@
 module Restic
   module Service
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: restic-service
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sylvain Joyeux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-26 00:00:00.000000000 Z
+date: 2018-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -100,6 +100,7 @@ files:
 - install-dev.sh
 - install.sh
 - lib/restic/service.rb
+- lib/restic/service/auto_update.rb
 - lib/restic/service/cli.rb
 - lib/restic/service/conf.rb
 - lib/restic/service/ssh_keys.rb