yobi 0.1.0

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "parallel_tests/tasks"
+
+require "standard/rake"
+
+task default: %i[spec standard]

data/lib/yobi/argv_builder.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Yobi
+  # Builds a Restic command's argv incrementally. Not an Array subclass:
+  # only the methods below are available; {#to_a} hands back the plain
+  # Array once building is done.
+  #
+  # @private
+  class ArgvBuilder
+    # Maps a flag name Symbol to its "--dashed-string" form, e.g.
+    # `:read_data_subset` to `"--read-data-subset"`.
+    FLAGS = Hash.new { |flags, name| flags[name] = "--#{name.to_s.tr("_", "-")}" }
+
+    # Maps a short flag name Symbol to its "-name" form, e.g. `:vv` to `"-vv"`.
+    SHORT_FLAGS = Hash.new { |flags, name| flags[name] = "-#{name}" }
+
+    def initialize
+      @argv = []
+      @end_of_options_called = false
+    end
+
+    # @return [Array<String>] the argv built so far
+    def to_a
+      @argv
+    end
+
+    # Appends one or more bare positional values. `nil` is dropped; an
+    # Enumerable is flattened in; everything else is converted with `#to_s`.
+    #
+    # @param values [Array<Object>]
+    # @return [self]
+    def append(*values)
+      values.each do |v|
+        case v
+        in nil
+        in String
+          @argv << v
+        in Enumerable
+          append(*v)
+        else
+          @argv << v.to_s
+        end
+      end
+      self
+    end
+
+    # Appends a flag, plus its value when one is given.
+    #
+    # @param name [Symbol] flag name, looked up in {FLAGS}
+    # @param value [Object, nil] the flag's value; omit for a boolean flag
+    # @return [self]
+    def flag(name, value = nil)
+      @argv << FLAGS[name]
+      @argv << value.to_s unless value.nil?
+      self
+    end
+
+    # Appends a repeatable flag once per value, e.g. `--host a --host b`.
+    #
+    # @param name [Symbol] flag name, looked up in {FLAGS}
+    # @param values [Array<Object>, Object, nil]
+    # @return [self]
+    def repeat_flag(name, values)
+      name = FLAGS[name]
+      array_of_strings(values).each { |value| @argv << name << value }
+      self
+    end
+
+    # Appends a short flag, e.g. `short_flag(:vv)` for `-vv`.
+    #
+    # @param name [Symbol] flag name, looked up in {SHORT_FLAGS}
+    # @return [self]
+    def short_flag(name)
+      @argv << SHORT_FLAGS[name]
+      self
+    end
+
+    # Appends Restic's `--` end-of-options marker.
+    #
+    # @return [self]
+    # @raise [RuntimeError] if called more than once on the same builder
+    def end_of_options
+      raise "end_of_options already called - Restic only honors the first \"--\"" if @end_of_options_called
+
+      @end_of_options_called = true
+      @argv << "--"
+      self
+    end
+
+    private
+
+    def array_of_strings(value)
+      if value.nil?
+        []
+      elsif value.respond_to?(:to_ary)
+        value.to_ary.map(&:to_s)
+      else
+        [value.to_s]
+      end
+    end
+  end
+end

data/lib/yobi/errors.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "json"
+require "delegate"
+
+module Yobi
+  # Base class for all errors raised by Yobi.
+  class Error < StandardError; end
+
+  # The `exit_error` message from a fatal Restic run, if one was printed.
+  # https://restic.readthedocs.io/en/stable/075_scripting.html#exit-errors
+  class ExitError < SimpleDelegator
+    # Matches a JSON line with `"message_type": "exit_error"`.
+    LINE_PATTERN = /"message_type"\s*:\s*"exit_error"/
+
+    # @param output [Yobi::ResticOutput]
+    # @return [Yobi::ExitError, nil] `nil` if no `exit_error` line is present
+    def self.from_output(output)
+      line = output.each_line.find { |candidate| LINE_PATTERN.match?(candidate) }
+      new(JSON.parse(line)) if line
+    end
+
+    # @return [String, nil]
+    def code
+      self["code"]
+    end
+
+    # @return [String, nil]
+    def message
+      self["message"]
+    end
+  end
+
+  # Raised when the installed Restic binary is older than
+  # {Yobi::Restic::MINIMUM_VERSION}.
+  class UnsupportedResticVersion < Error
+    # @return [String]
+    attr_reader :installed_version, :minimum_version
+
+    # @param installed_version [String]
+    # @param minimum_version [String]
+    def initialize(installed_version:, minimum_version:)
+      @installed_version = installed_version
+      @minimum_version = minimum_version
+      super("Restic #{installed_version} does not meet the minimum supported version #{minimum_version}")
+    end
+  end
+
+  # Raised when the Restic binary itself can't be found or executed.
+  class ResticNotFound < Error
+    # @return [String]
+    attr_reader :restic_path
+    # @return [Array<String>]
+    attr_reader :argv
+
+    # @param restic_path [String]
+    # @param argv [Array<String>]
+    def initialize(restic_path:, argv:)
+      @restic_path = restic_path
+      @argv = argv
+      super("Restic binary not found: #{restic_path.inspect} (tried to run #{argv.inspect})")
+    end
+  end
+
+  # Raised when Restic exits with a failure code. Base class for the
+  # specific typed errors below.
+  class ResticExecutionError < Error
+    # @return [Hash] the raw `{exit_code:, output:, argv:}` execution result
+    attr_reader :execution
+    # @return [Yobi::ExitError, nil]
+    attr_reader :exit_error
+
+    # @param execution [Hash]
+    def initialize(execution)
+      @execution = execution
+      @exit_error = Yobi::ExitError.from_output(execution[:output])
+      super(exit_error&.message || "Restic exited with status #{execution[:exit_code]}: #{execution[:output]}")
+    end
+  end
+
+  # Raised when the repository does not exist (Restic exit code 10).
+  class RepositoryNotFound < ResticExecutionError; end
+
+  # Raised when the repository is locked by another process (Restic exit code 11).
+  class RepositoryLocked < ResticExecutionError; end
+
+  # Raised when the repository password is incorrect (Restic exit code 12).
+  class AuthenticationFailed < ResticExecutionError; end
+
+  # Raised when Restic exits with a failure code not otherwise classified above.
+  class ResticCommandFailed < ResticExecutionError; end
+
+  # Raised by {Yobi::Repository#mount} when Restic doesn't report itself
+  # ready within `ready_timeout:` seconds.
+  class MountTimeout < Error
+    # @return [Array<String>]
+    attr_reader :argv
+    # @return [Numeric]
+    attr_reader :timeout
+
+    # @param argv [Array<String>]
+    # @param timeout [Numeric]
+    def initialize(argv:, timeout:)
+      @argv = argv
+      @timeout = timeout
+      super("Restic mount didn't report readiness within #{timeout}s (tried to run #{argv.inspect})")
+    end
+  end
+end

data/lib/yobi/io_handle.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Yobi
+  # A live Restic process streaming raw bytes to its own stdout, returned by
+  # {Yobi::Restic#run_dump} when no block is given. Satisfies Rack's Body
+  # contract (`#each`, `#close`).
+  #
+  # Unlike the block form, nothing closes this automatically. Call {#close}
+  # yourself once done reading, in an `ensure`, so it still runs if reading
+  # raises:
+  #
+  # @example
+  #   handle = repo.dump(snapshot_id: "latest", file: "/etc/hosts")
+  #   begin
+  #     IO.copy_stream(handle.io, "/tmp/hosts")
+  #   ensure
+  #     handle.close
+  #   end
+  class IOHandle
+    # @return [IO] the readable end of the pipe Restic's stdout is wired to
+    attr_reader :io
+    # @return [Integer] the Restic process's pid
+    attr_reader :pid
+
+    # @param io [IO]
+    # @param pid [Integer]
+    # @param output_file [File]
+    # @param argv [Array<String>]
+    def initialize(io, pid:, output_file:, argv:)
+      @io = io
+      @pid = pid
+      @output_file = output_file
+      @argv = argv
+      @closed = false
+    end
+
+    # @return [Boolean]
+    def closed?
+      @closed
+    end
+
+    # Reaps the Restic process and raises based on its exit code. Safe to
+    # call more than once.
+    #
+    # @return [void]
+    def close
+      return if @closed
+
+      @closed = true
+      @io.close unless @io.closed?
+      _, status = Process.wait2(@pid)
+      Restic.dispatch(exit_code: status.exitstatus, output: Yobi::ResticOutput.new(@output_file), argv: @argv)
+    end
+
+    # Yields binary-safe chunks of {#io} until EOF, then calls {#close}.
+    #
+    # @yieldparam chunk [String]
+    # @return [void]
+    def each
+      loop do
+        yield @io.readpartial(64 * 1024)
+      rescue EOFError
+        break
+      end
+      close
+    end
+  end
+end

data/lib/yobi/mount.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Yobi
+  # A live Restic mount process, returned by {Yobi::Restic#run_mount} once
+  # Restic has reported itself ready. The mounted filesystem itself is
+  # browsed with ordinary file I/O at {#mountpoint}; this object only
+  # manages the Restic process's lifetime.
+  class Mount
+    # @return [String] the path the repository is mounted at
+    attr_reader :mountpoint
+
+    # @param wait_thr [Process::Waiter] as returned by `Open3.popen2e`
+    # @param mountpoint [String]
+    # @param output [IO]
+    # @param output_file [File]
+    # @param argv [Array<String>]
+    def initialize(wait_thr:, mountpoint:, output:, output_file:, argv:)
+      @wait_thr = wait_thr
+      @mountpoint = mountpoint
+      @output = output
+      @output_file = output_file
+      @argv = argv
+      @stopped = false
+    end
+
+    # @return [Integer] the Restic process's pid
+    def pid
+      @wait_thr.pid
+    end
+
+    # @return [Boolean]
+    def stopped?
+      @stopped
+    end
+
+    # Sends the Restic process SIGINT, waits for it to unmount and exit,
+    # then raises based on its exit code. Safe to call more than once, and
+    # safe to call after the mount has already been stopped externally
+    # (e.g. via `kill -INT` or the OS's own `umount`/`fusermount`).
+    #
+    # @return [void]
+    def stop
+      return if @stopped
+
+      @stopped = true
+
+      begin
+        Process.kill("INT", pid)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      drain_remaining_output
+      status = @wait_thr.value
+      Restic.dispatch(exit_code: status.exitstatus, output: Yobi::ResticOutput.new(@output_file), argv: @argv)
+    end
+
+    private
+
+    def drain_remaining_output
+      @output.each_line { |line| @output_file.write(line) }
+    rescue IOError, Errno::EBADF
+      nil
+    end
+  end
+end

data/lib/yobi/repository/backup.rb

@@ -0,0 +1,310 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "delegate"
+require "shellwords"
+
+module Yobi
+  class Repository
+    # `restic backup`: creates a new snapshot from a source path.
+    #
+    # @param source [String, Array] a path to back up, or a
+    #   `[:stdin_from_command, command]`/`[:stdin_from_command, command, filename]`
+    #   tuple. Restic spawns and executes `command` itself (a String,
+    #   tokenized with `Shellwords.split`, or an Array of already-discrete
+    #   arguments), capturing its stdout as the backup content
+    # @param excludes [Array<String>, String] exclude files matching these glob patterns
+    # @param exclude_files [Array<String>, String] path(s) to file(s) listing exclude patterns, one per line
+    # @param exclude_if_present [Array<String>, String] skip a directory containing a marker file with this name (optionally `name:content`)
+    # @param exclude_larger_than [String, nil] skip files larger than this size, e.g. `"1G"`
+    # @param files_from [Array<String>, String] read the files/dirs to back up from a file, one per line
+    # @param files_from_raw [Array<String>, String] like `files_from`, NUL-separated
+    # @param files_from_verbatim [Array<String>, String] like `files_from`, paths taken literally with no glob expansion
+    # @param iexcludes [Array<String>, String] like `excludes`, case-insensitive
+    # @param iexclude_files [Array<String>, String] like `exclude_files`, case-insensitive
+    # @param tags [Array<String>, String] tags to attach to the new snapshot
+    # @param dry_run [Boolean] report what would happen without doing it
+    # @param exclude_caches [Boolean] skip directories containing a CACHEDIR.TAG marker
+    # @param exclude_cloud_files [Boolean] skip files not fully present on disk (e.g. OneDrive placeholders)
+    # @param force [Boolean] back up unchanged files instead of skipping them
+    # @param group_by [String, nil] grouping used to find the parent snapshot, e.g. `"host,paths"`
+    # @param host [String, nil] hostname to record on the new snapshot, instead of the OS hostname
+    # @param ignore_ctime [Boolean] relax change detection to ignore ctime
+    # @param ignore_inode [Boolean] relax change detection to ignore inode number
+    # @param no_scan [Boolean] skip the pre-backup scan (disables percentage progress)
+    # @param one_file_system [Boolean] don't cross filesystem boundaries
+    # @param parent [String, nil] snapshot ID to use as the parent instead of the latest one
+    # @param read_concurrency [Integer, nil] number of concurrent file reads
+    # @param skip_if_unchanged [Boolean] don't create a snapshot if nothing changed
+    # @param time [String, nil] timestamp to record instead of now
+    # @param verbose [Boolean] stream a {Yobi::BackupVerboseStatus} per file to the block
+    # @param with_atime [Boolean] also store files' access times
+    # @yieldparam message [Yobi::BackupStatus, Yobi::BackupError, Yobi::BackupVerboseStatus]
+    # @return [Yobi::BackupOutcome]
+    def backup(source:, excludes: [], exclude_files: [], exclude_if_present: [], exclude_larger_than: nil,
+      files_from: [], files_from_raw: [], files_from_verbatim: [], iexcludes: [], iexclude_files: [],
+      tags: [], dry_run: false, exclude_caches: false, exclude_cloud_files: false, force: false,
+      group_by: nil, host: nil, ignore_ctime: false, ignore_inode: false, no_scan: false,
+      one_file_system: false, parent: nil, read_concurrency: nil, skip_if_unchanged: false,
+      time: nil, verbose: false, with_atime: false, &block)
+      argv = build_argv("backup") do |a|
+        a.repeat_flag(:exclude, excludes)
+        a.repeat_flag(:exclude_file, exclude_files)
+        a.repeat_flag(:exclude_if_present, exclude_if_present)
+        a.flag(:exclude_larger_than, exclude_larger_than) unless exclude_larger_than.nil?
+        a.repeat_flag(:files_from, files_from)
+        a.repeat_flag(:files_from_raw, files_from_raw)
+        a.repeat_flag(:files_from_verbatim, files_from_verbatim)
+        a.repeat_flag(:iexclude, iexcludes)
+        a.repeat_flag(:iexclude_file, iexclude_files)
+        a.repeat_flag(:tag, tags)
+        a.flag(:dry_run) if dry_run
+        a.flag(:exclude_caches) if exclude_caches
+        a.flag(:exclude_cloud_files) if exclude_cloud_files
+        a.flag(:force) if force
+        a.flag(:group_by, group_by) unless group_by.nil?
+        a.flag(:host, host) unless host.nil?
+        a.flag(:ignore_ctime) if ignore_ctime
+        a.flag(:ignore_inode) if ignore_inode
+        a.flag(:no_scan) if no_scan
+        a.flag(:one_file_system) if one_file_system
+        a.flag(:parent, parent) unless parent.nil?
+        a.flag(:read_concurrency, read_concurrency) unless read_concurrency.nil?
+        a.flag(:skip_if_unchanged) if skip_if_unchanged
+        a.flag(:time, time) unless time.nil?
+        a.short_flag(:v) if verbose
+        a.flag(:with_atime) if with_atime
+        case source
+        in String => path
+          a.end_of_options.append(path)
+        in [:stdin_from_command, command]
+          a.flag(:stdin_from_command)
+          a.end_of_options.append(tokenize(command))
+        in [:stdin_from_command, command, String => filename]
+          a.flag(:stdin_from_command)
+          a.flag(:stdin_filename, filename)
+          a.end_of_options.append(tokenize(command))
+        end
+      end
+
+      execution = if block
+        run_restic(argv) { |raw| dispatch_backup_message(raw, &block) }
+      else
+        run_restic(argv)
+      end
+      BackupOutcome.new(execution[:exit_code], execution[:output])
+    end
+
+    private
+
+    def dispatch_backup_message(raw)
+      case raw["message_type"]
+      when "status"
+        yield Yobi::BackupStatus.new(raw)
+      when "error"
+        yield Yobi::BackupError.new(raw)
+      when "verbose_status"
+        yield Yobi::BackupVerboseStatus.new(raw)
+      end
+    end
+
+    def tokenize(command)
+      case command
+      in String => value
+        Shellwords.split(value)
+      in Array => values
+        values.map(&:to_s)
+      end
+    end
+  end
+
+  # The outcome of one {Yobi::Repository#backup} call.
+  class BackupOutcome < Struct.new(:exit_code, :output)
+    # Matches Restic's own log line format for stderr relayed from a
+    # `source: [:stdin_from_command, ...]` command's own subprocess.
+    COMMAND_OUTPUT_LINE_PATTERN = /\Asubprocess [^:]+: (.*)/
+
+    # @return [Boolean]
+    def success?
+      exit_code.zero?
+    end
+
+    # @return [Boolean] true if the backup completed with some files skipped
+    def partial?
+      exit_code == 3
+    end
+
+    # @return [Enumerable<Yobi::BackupError>]
+    def errors
+      @errors ||= BackupErrors.new(output)
+    end
+
+    # The `source: [:stdin_from_command, ...]` subprocess's own stderr, if any, de-prefixed.
+    #
+    # @return [Array<String>]
+    def command_output
+      @command_output ||= output.each_line.filter_map do |line|
+        match = COMMAND_OUTPUT_LINE_PATTERN.match(line)
+        match[1] if match
+      end
+    end
+
+    # @return [Hash] Restic's own `"summary"` fields, with `"backup_start"`/`"backup_end"` parsed into `Time`
+    def report
+      @report ||= begin
+        hash = summary_hash
+        hash.merge(
+          "backup_start" => (Time.parse(hash["backup_start"]) if hash["backup_start"]),
+          "backup_end" => (Time.parse(hash["backup_end"]) if hash["backup_end"])
+        )
+      end
+    end
+
+    private
+
+    def summary_hash
+      line = output.last_line
+      hash = line && JSON.parse(line)
+      if hash && hash["message_type"] == "summary"
+        hash
+      else
+        find_summary || {}
+      end
+    end
+
+    def find_summary
+      output.each_line do |line|
+        next if line.strip.empty?
+
+        hash = JSON.parse(line)
+        return hash if hash["message_type"] == "summary"
+      end
+
+      nil
+    end
+  end
+
+  # One `"status"` message from a live backup run.
+  # https://restic.readthedocs.io/en/stable/075_scripting.html#status
+  class BackupStatus < SimpleDelegator
+    # @return [Float, nil]
+    def percent_done
+      self["percent_done"]
+    end
+
+    # @return [Integer, nil]
+    def total_files
+      self["total_files"]
+    end
+
+    # @return [Integer, nil]
+    def files_done
+      self["files_done"]
+    end
+
+    # @return [Integer, nil]
+    def total_bytes
+      self["total_bytes"]
+    end
+
+    # @return [Integer, nil]
+    def bytes_done
+      self["bytes_done"]
+    end
+
+    # @return [Array<String>]
+    def current_files
+      self["current_files"] || []
+    end
+
+    # @return [Integer]
+    def error_count
+      self["error_count"] || 0
+    end
+  end
+
+  # One `"error"` message from a backup run.
+  # https://restic.readthedocs.io/en/stable/075_scripting.html#error
+  class BackupError < SimpleDelegator
+    # @return [String, nil]
+    def message
+      dig("error", "message")
+    end
+
+    # @return [String, nil]
+    def during
+      self["during"]
+    end
+
+    # @return [String, nil]
+    def item
+      self["item"]
+    end
+  end
+
+  # Enumerable over every {Yobi::BackupError} in a run.
+  class BackupErrors
+    include Enumerable
+
+    # @param output [Yobi::ResticOutput]
+    def initialize(output)
+      @output = output
+    end
+
+    # @yieldparam error [Yobi::BackupError]
+    # @return [Enumerator] if no block is given
+    def each
+      return enum_for(:each) unless block_given?
+
+      @output.index["error"].each do |offset|
+        line = @output.read_line_at(offset)
+        yield BackupError.new(JSON.parse(line))
+      end
+    end
+  end
+
+  # One `"verbose_status"` message from a backup run, one per file. Only
+  # emitted when `verbose: true` is passed to {Yobi::Repository#backup}.
+  class BackupVerboseStatus < SimpleDelegator
+    # @return [String]
+    def action
+      self["action"]
+    end
+
+    # @return [String]
+    def item
+      self["item"]
+    end
+
+    # @return [Integer]
+    def duration
+      self["duration"]
+    end
+
+    # @return [Integer]
+    def data_size
+      self["data_size"]
+    end
+
+    # @return [Integer]
+    def data_size_in_repo
+      self["data_size_in_repo"]
+    end
+
+    # @return [Integer]
+    def metadata_size
+      self["metadata_size"]
+    end
+
+    # @return [Integer]
+    def metadata_size_in_repo
+      self["metadata_size_in_repo"]
+    end
+
+    # @return [Integer]
+    def total_files
+      self["total_files"]
+    end
+  end
+end