yobi 0.1.0

data/lib/yobi/restic.rb

@@ -0,0 +1,341 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "open3"
+require "json"
+
+module Yobi
+  # A specific Restic binary, plus its process-level settings that apply
+  # regardless of which repository is being operated on.
+  # Repository-specific identity (url, credentials) lives on
+  # {Yobi::Repository} instead.
+  class Restic
+    # The oldest Restic version {#run}/{#run_dump}/{#run_mount} verify the
+    # installed binary meets before executing a real command. See
+    # {Yobi::UnsupportedResticVersion}.
+    MINIMUM_VERSION = Gem::Version.new("0.17.1")
+
+    # Env vars {#inspect} shows in the clear; anything else is redacted.
+    ALLOWED_ENV_VARS = %w[
+      RESTIC_REPOSITORY RESTIC_CACHE_DIR RESTIC_COMPRESSION RESTIC_PACK_SIZE
+      RESTIC_READ_CONCURRENCY RESTIC_HOST RESTIC_PROGRESS_FPS RESTIC_CACERT
+      RESTIC_TLS_CLIENT_CERT RESTIC_KEY_HINT TMPDIR TMP AWS_DEFAULT_REGION
+      AWS_SHARED_CREDENTIALS_FILE AZURE_ENDPOINT_SUFFIX AZURE_FORCE_CLI_CREDENTIAL
+      RCLONE_BWLIMIT
+    ].to_set.freeze
+
+    # @return [String] path to the Restic binary
+    attr_reader :restic_path
+    # @return [String, nil] `$RESTIC_CACHE_DIR`
+    attr_accessor :cache_dir
+    # @return [String, nil] `$RESTIC_COMPRESSION`
+    attr_accessor :compression
+    # @return [Integer, nil] `$RESTIC_PACK_SIZE`
+    attr_accessor :pack_size
+    # @return [Integer, nil] `$RESTIC_READ_CONCURRENCY`
+    attr_accessor :read_concurrency
+    # @return [String, nil] `$RESTIC_HOST`
+    attr_accessor :host
+    # @return [Integer, nil] `$RESTIC_PROGRESS_FPS`
+    attr_accessor :progress_fps
+    # @return [String, nil] `$RESTIC_CACERT`
+    attr_accessor :cacert
+    # @return [String, nil] `$RESTIC_TLS_CLIENT_CERT`
+    attr_accessor :tls_client_cert
+    # @return [String, nil] `$RESTIC_KEY_HINT`
+    attr_accessor :key_hint
+    # @return [String, nil] `--limit-download`
+    attr_accessor :limit_download
+    # @return [String, nil] `--limit-upload`
+    attr_accessor :limit_upload
+    # @return [String, nil] `--retry-lock`
+    attr_accessor :retry_lock
+    # @return [Boolean] `--no-lock`
+    attr_accessor :no_lock
+    # @return [Boolean] `--no-cache`
+    attr_accessor :no_cache
+    # @return [Boolean] `--cleanup-cache`
+    attr_accessor :cleanup_cache
+    # @return [Boolean] `--no-extra-verify`
+    attr_accessor :no_extra_verify
+    # @return [String, nil] `--stuck-request-timeout`
+    attr_accessor :stuck_request_timeout
+    # @return [Array<String>] `--option`, one per element
+    attr_accessor :options
+    # @return [String, nil] `--http-user-agent`
+    attr_accessor :http_user_agent
+    # @return [Boolean] `--quiet`
+    attr_accessor :quiet
+
+    # @param restic_path [String, nil] defaults to `$RESTIC_PATH`, then `"restic"`
+    # @param env [Hash{String => String}] extra env vars, merged over the named settings below
+    # @param cache_dir [String, nil]
+    # @param compression [String, nil]
+    # @param pack_size [Integer, nil]
+    # @param read_concurrency [Integer, nil]
+    # @param host [String, nil]
+    # @param progress_fps [Integer, nil]
+    # @param cacert [String, nil]
+    # @param tls_client_cert [String, nil]
+    # @param key_hint [String, nil]
+    # @param limit_download [String, nil]
+    # @param limit_upload [String, nil]
+    # @param retry_lock [String, nil]
+    # @param no_lock [Boolean]
+    # @param no_cache [Boolean]
+    # @param cleanup_cache [Boolean]
+    # @param no_extra_verify [Boolean]
+    # @param stuck_request_timeout [String, nil]
+    # @param options [Array<String>]
+    # @param http_user_agent [String, nil]
+    # @param quiet [Boolean]
+    def initialize(restic_path = nil, env: {}, cache_dir: nil, compression: nil, pack_size: nil,
+      read_concurrency: nil, host: nil, progress_fps: nil, cacert: nil, tls_client_cert: nil,
+      key_hint: nil, limit_download: nil, limit_upload: nil, retry_lock: nil, no_lock: false,
+      no_cache: false, cleanup_cache: false, no_extra_verify: false, stuck_request_timeout: nil,
+      options: [], http_user_agent: nil, quiet: false)
+      @restic_path = restic_path || ENV.fetch("RESTIC_PATH", "restic")
+      @env = env
+      @cache_dir = cache_dir
+      @compression = compression
+      @pack_size = pack_size
+      @read_concurrency = read_concurrency
+      @host = host
+      @progress_fps = progress_fps
+      @cacert = cacert
+      @tls_client_cert = tls_client_cert
+      @key_hint = key_hint
+      @limit_download = limit_download
+      @limit_upload = limit_upload
+      @retry_lock = retry_lock
+      @no_lock = no_lock
+      @no_cache = no_cache
+      @cleanup_cache = cleanup_cache
+      @no_extra_verify = no_extra_verify
+      @stuck_request_timeout = stuck_request_timeout
+      @options = options
+      @http_user_agent = http_user_agent
+      @quiet = quiet
+    end
+
+    # The env-var-backed settings above, computed fresh from their current
+    # accessor values on every call. `env:` given at construction wins over
+    # any of these on key collision.
+    #
+    # @return [Hash{String => String}]
+    def env
+      {
+        "RESTIC_CACHE_DIR" => cache_dir,
+        "RESTIC_COMPRESSION" => compression,
+        "RESTIC_PACK_SIZE" => pack_size&.to_s,
+        "RESTIC_READ_CONCURRENCY" => read_concurrency&.to_s,
+        "RESTIC_HOST" => host,
+        "RESTIC_PROGRESS_FPS" => progress_fps&.to_s,
+        "RESTIC_CACERT" => cacert,
+        "RESTIC_TLS_CLIENT_CERT" => tls_client_cert,
+        "RESTIC_KEY_HINT" => key_hint
+      }.compact.merge(@env)
+    end
+
+    # Appends the CLI-only global flags (the ones above with no env var
+    # equivalent) to a builder. Called from both {#build_argv} and
+    # `Repository#build_argv`.
+    #
+    # @param a [Yobi::ArgvBuilder]
+    # @return [void]
+    def append_global_flags(a)
+      a.flag(:limit_download, limit_download) unless limit_download.nil?
+      a.flag(:limit_upload, limit_upload) unless limit_upload.nil?
+      a.flag(:retry_lock, retry_lock) unless retry_lock.nil?
+      a.flag(:no_lock) if no_lock
+      a.flag(:no_cache) if no_cache
+      a.flag(:cleanup_cache) if cleanup_cache
+      a.flag(:no_extra_verify) if no_extra_verify
+      a.flag(:stuck_request_timeout, stuck_request_timeout) unless stuck_request_timeout.nil?
+      a.repeat_flag(:option, options)
+      a.flag(:http_user_agent, http_user_agent) unless http_user_agent.nil?
+      a.flag(:quiet) if quiet
+    end
+
+    # @return [String]
+    def inspect
+      "#<#{self.class} restic_path=#{restic_path.inspect} env=#{redacted_env.inspect}>"
+    end
+
+    # `restic version`: the installed binary's own version info.
+    #
+    # @return [Hash] `"version"`, `"go_version"`, `"go_os"`, `"go_arch"`
+    def version
+      execution = run(build_argv("version"), skip_version_check: true)
+      JSON.parse(execution[:output].to_s)
+    end
+
+    # `restic cache`: lists and optionally cleans local cache directories.
+    # Not repository-scoped.
+    #
+    # @param cleanup [Boolean] `--cleanup`
+    # @param max_age [String, nil] `--max-age`
+    # @param no_size [Boolean] `--no-size`
+    # @return [true]
+    def cache(cleanup: false, max_age: nil, no_size: false)
+      argv = build_argv("cache") do |a|
+        a.flag(:cleanup) if cleanup
+        a.flag(:max_age, max_age) unless max_age.nil?
+        a.flag(:no_size) if no_size
+      end
+      run(argv)
+      true
+    end
+
+    # Runs argv against this Restic binary, merging extra_env with this
+    # instance's own global env.
+    #
+    # @param argv [Array<String>]
+    # @param extra_env [Hash{String => String}]
+    # @param skip_version_check [Boolean] used internally by {#version} to avoid recursing into {#ensure_minimum_version!}
+    # @yieldparam message [Hash] a parsed JSON line, as the command runs
+    # @return [Hash] `{exit_code:, output:, argv:}` on exit code 0/3
+    # @raise [Yobi::RepositoryNotFound, Yobi::RepositoryLocked, Yobi::AuthenticationFailed, Yobi::ResticCommandFailed]
+    def run(argv, extra_env: {}, skip_version_check: false, &block)
+      ensure_minimum_version! unless skip_version_check
+      execution = if block
+        execute_with_streaming(argv, extra_env, &block)
+      else
+        execute(argv, extra_env)
+      end
+      self.class.dispatch(execution)
+    end
+
+    # @param execution [Hash] `{exit_code:, output:, argv:}`
+    # @return [Hash] `execution`, unchanged, on exit code 0/3
+    # @raise [Yobi::RepositoryNotFound, Yobi::RepositoryLocked, Yobi::AuthenticationFailed, Yobi::ResticCommandFailed]
+    def self.dispatch(execution)
+      case execution[:exit_code]
+      when 0, 3
+        execution
+      when 10
+        raise Yobi::RepositoryNotFound, execution
+      when 11
+        raise Yobi::RepositoryLocked, execution
+      when 12
+        raise Yobi::AuthenticationFailed, execution
+      else
+        raise Yobi::ResticCommandFailed, execution
+      end
+    end
+
+    # For commands whose success output is raw bytes with no JSON message
+    # framing (currently only `Repository#dump`). Spawns Restic with
+    # stdout wired to a pipe.
+    #
+    # Without a block, returns a {Yobi::IOHandle} immediately; closing,
+    # reaping, and exit-code dispatch are the caller's own responsibility.
+    # With one, yields the pipe's read end; it's always closed and the
+    # process always reaped once the block returns or raises, before any
+    # exit-code dispatch runs.
+    #
+    # @param argv [Array<String>]
+    # @param extra_env [Hash{String => String}]
+    # @yieldparam io [IO]
+    # @return [Yobi::IOHandle] if no block is given
+    # @return [Object] the block's own return value, otherwise
+    def run_dump(argv, extra_env: {})
+      ensure_minimum_version!
+      file = Tempfile.new("yobi-restic-output")
+      file.unlink
+      read_end, write_end = IO.pipe
+      read_end.binmode
+
+      pid = Process.spawn(env.merge(extra_env), restic_path, *argv, out: write_end, err: file)
+      write_end.close
+
+      return Yobi::IOHandle.new(read_end, pid: pid, output_file: file, argv: argv) unless block_given?
+
+      begin
+        result = yield read_end
+      ensure
+        read_end.close unless read_end.closed?
+        _, status = Process.wait2(pid)
+      end
+
+      self.class.dispatch(exit_code: status.exitstatus, output: Yobi::ResticOutput.new(file), argv: argv)
+      result
+    rescue Errno::ENOENT
+      file&.close
+      raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
+    end
+
+    # Verifies the installed Restic binary meets {MINIMUM_VERSION}, once
+    # per instance (memoized). {#run}/{#run_dump}/{#run_mount} call this
+    # automatically before executing a real command; public so a caller
+    # can also call it explicitly to fail fast.
+    #
+    # @return [void]
+    # @raise [Yobi::UnsupportedResticVersion]
+    def ensure_minimum_version!
+      return if defined?(@version_checked)
+      @version_checked = true
+
+      installed = Gem::Version.new(version["version"])
+      return if installed >= MINIMUM_VERSION
+
+      raise Yobi::UnsupportedResticVersion.new(installed_version: installed.to_s, minimum_version: MINIMUM_VERSION.to_s)
+    end
+
+    private
+
+    def build_argv(*base)
+      builder = ArgvBuilder.new
+      builder.append(*base)
+      builder.flag(:json)
+      append_global_flags(builder)
+      yield builder if block_given?
+      builder.to_a
+    end
+
+    def redacted_env
+      env.each_with_object({}) do |(key, value), filtered|
+        filtered[key] = if ALLOWED_ENV_VARS.include?(key)
+          value
+        else
+          "[FILTERED]"
+        end
+      end
+    end
+
+    def execute(argv, extra_env)
+      file = Tempfile.new("yobi-restic-output")
+      file.unlink
+      pid = Process.spawn(env.merge(extra_env), restic_path, *argv, out: file, err: file)
+      _, status = Process.wait2(pid)
+      {exit_code: status.exitstatus, output: Yobi::ResticOutput.new(file), argv: argv}
+    rescue Errno::ENOENT
+      file&.close
+      raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
+    end
+
+    def execute_with_streaming(argv, extra_env)
+      file = Tempfile.new("yobi-restic-output")
+      file.unlink
+
+      Open3.popen2e(env.merge(extra_env), restic_path, *argv) do |stdin, merged_output, wait_thr|
+        stdin.close
+        merged_output.each_line do |line|
+          file.write(line)
+          parsed = parse_streamed_line(line)
+          yield parsed if parsed
+        end
+        {exit_code: wait_thr.value.exitstatus, output: Yobi::ResticOutput.new(file), argv: argv}
+      end
+    rescue Errno::ENOENT
+      file&.close
+      raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
+    end
+
+    def parse_streamed_line(line)
+      JSON.parse(line)
+    rescue JSON::ParserError
+      nil
+    end
+  end
+end

data/lib/yobi/restic_output.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Yobi
+  # Captures a line's message_type without fully parsing it as JSON.
+  #
+  # @private
+  MESSAGE_TYPE_LINE_PATTERN = /"message_type"\s*:\s*"(\w+)"/
+
+  # Wraps the tempfile a repository operation's stdout+stderr are both
+  # written to, with random access so a caller asking only for the last
+  # line (see {#last_line}) doesn't have to read the entire output.
+  #
+  # @private
+  class ResticOutput
+    # Default read window size, in bytes, for {#last_line}.
+    DEFAULT_TAIL_CHUNK_SIZE = 4096
+
+    # @param file [File]
+    # @param tail_chunk_size [Integer] read window size for {#last_line}
+    def initialize(file, tail_chunk_size: DEFAULT_TAIL_CHUNK_SIZE)
+      @file = file
+      @tail_chunk_size = tail_chunk_size
+    end
+
+    # Byte offsets of every line, grouped by message_type. Built once and
+    # memoized.
+    #
+    # @return [Hash{String => Array<Integer>}]
+    def index
+      @index ||= build_index
+    end
+
+    # @param offset [Integer] a byte offset, as recorded by {#index}
+    # @return [String] the line starting at that offset
+    def read_line_at(offset)
+      @file.seek(offset)
+      @file.gets
+    end
+
+    # The last non-blank line, found by seeking backward from the end of
+    # the file.
+    #
+    # @return [String, nil]
+    def last_line
+      size = @file.size
+      return nil if size.zero?
+
+      window = @tail_chunk_size
+      loop do
+        read_size = [window, size].min
+        @file.seek(size - read_size)
+        lines = @file.read(read_size).each_line.to_a
+        lines.shift if read_size < size
+
+        line = lines.reverse_each.find { |candidate| !candidate.strip.empty? }
+        return line.strip if line
+        return nil if read_size == size
+
+        window *= 2
+      end
+    end
+
+    # @yieldparam line [String]
+    # @return [Enumerator] if no block is given
+    def each_line(&block)
+      return enum_for(:each_line) unless block_given?
+
+      @file.rewind
+      @file.each_line(&block)
+    end
+
+    # @return [String] the full captured output
+    def to_s
+      @file.rewind
+      @file.read
+    end
+
+    private
+
+    def build_index
+      index = Hash.new { |hash, message_type| hash[message_type] = [] }
+      @file.rewind
+      loop do
+        offset = @file.pos
+        line = @file.gets
+        break unless line
+
+        match = MESSAGE_TYPE_LINE_PATTERN.match(line)
+        index[match[1]] << offset if match
+      end
+      index
+    end
+  end
+end

data/lib/yobi/snapshot.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "time"
+require "delegate"
+
+module Yobi
+  # One snapshot, as Restic reports it. Shared across {Yobi::Repository#snapshots},
+  # {Yobi::Repository#ls}, and {Yobi::Repository#forget}'s keep/remove entries.
+  class Snapshot < SimpleDelegator
+    # @return [String]
+    def id
+      self["id"]
+    end
+
+    # @return [String]
+    def short_id
+      self["short_id"]
+    end
+
+    # @return [Time]
+    def time
+      @time ||= Time.parse(self["time"])
+    end
+
+    # @return [String]
+    def host
+      self["hostname"]
+    end
+
+    # @return [Array<String>]
+    def tags
+      @tags ||= self["tags"] || []
+    end
+
+    # @return [Array<String>]
+    def paths
+      self["paths"]
+    end
+
+    # @return [String, nil]
+    def parent_id
+      self["parent"]
+    end
+  end
+end

data/lib/yobi/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# A Ruby library for the Restic backup tool, wrapping the `restic` CLI in
+# plain Ruby objects instead of shelling out to flags and raw JSON by hand.
+module Yobi
+  # @return [String]
+  VERSION = "0.1.0"
+end

data/lib/yobi.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "yobi/version"
+require_relative "yobi/errors"
+require_relative "yobi/argv_builder"
+require_relative "yobi/restic_output"
+require_relative "yobi/io_handle"
+require_relative "yobi/mount"
+require_relative "yobi/restic"
+require_relative "yobi/snapshot"
+require_relative "yobi/repository"
+
+Dir[File.join(__dir__, "yobi", "repository", "*.rb")].sort.each { |file| require_relative file }