yobi 0.3.1 → 1.1.0

data/lib/yobi/repository.rb

@@ -7,24 +7,48 @@ module Yobi
   # One Restic repository. Every Restic subcommand that operates on a
   # repository is a method here.
   class Repository
-    # @return [String] the repository location
+    # The repository location.
     attr_reader :url
-    # @return [String, Array, Symbol, #call] the repository's encryption password, as given to {#initialize}
+    # The repository's encryption password, as given to #initialize.
     attr_reader :password
-    # @return [Hash, #call] the storage backend's own credentials, as given to {#initialize}
+    # The storage backend's own credentials, as given to #initialize.
     attr_reader :backend_credentials
 
-    # @param url [String] the repository location, e.g. `"s3:s3.amazonaws.com/bucket"`
-    # @param password [String, Array, Symbol, #call] a literal password; a
-    #   `[:command, "..."]`/`[:file, "..."]` tuple, resolved natively by Restic
-    #   itself; `:insecure_no_password`; or anything responding to `#call`
-    #   (invoked fresh immediately before every Restic invocation)
-    # @param backend_credentials [Hash{String => String}, #call] the storage
-    #   backend's own env vars (`AWS_*`/`AZURE_*`/etc.), or anything
-    #   responding to `#call` returning such a Hash
-    # @param restic [Yobi::Restic, String, nil] a `Restic` instance to share,
-    #   a bare Restic binary path, or `nil` to create a default one
-    # @raise [ArgumentError] if `password:` is `nil`, or if `password:`/`backend_credentials:`/`restic:` is an invalid shape
+    # Sets #url and re-extracts any REST credentials embedded in it.
+    def url=(value)
+      @url, @extracted_rest_credentials = extract_rest_credentials(value)
+    end
+
+    # Sets #password. Validated on assignment with the same shapes #initialize accepts.
+    def password=(value)
+      validate_password_shape!(value)
+      @password = value
+    end
+
+    # Sets #backend_credentials. Validated on assignment with the same shapes #initialize accepts.
+    def backend_credentials=(value)
+      validate_backend_credentials_shape!(value)
+      @backend_credentials = value
+    end
+
+    # Builds a repository handle. Does not touch the remote or spawn Restic.
+    #
+    # +url+ is a String repository location, e.g. <tt>"s3:s3.amazonaws.com/bucket"</tt>.
+    #
+    # +password+ is the repository's encryption password. Accepts one of:
+    # a literal String; a <tt>[:command, "..."]</tt> or <tt>[:file, "..."]</tt>
+    # tuple, resolved natively by Restic itself; <tt>:insecure_no_password</tt>;
+    # or any object responding to +#call+ (invoked fresh immediately before every
+    # Restic invocation).
+    #
+    # +backend_credentials+ is the storage backend's own env vars
+    # (+AWS_*+/+AZURE_*+/etc.) as a Hash, or a callable returning such a Hash.
+    #
+    # +restic+ is a Yobi::Restic instance to share, a bare Restic binary path,
+    # or +nil+ to create a default one.
+    #
+    # Raises ArgumentError when +password:+ is +nil+, or when +password:+,
+    # +backend_credentials:+, or +restic:+ has an invalid shape.
     def initialize(url:, password:, backend_credentials: {}, restic: nil)
       validate_password_shape!(password)
       validate_backend_credentials_shape!(backend_credentials)
@@ -35,22 +59,42 @@ module Yobi
       @restic = initialize_restic(restic)
     end
 
-    # @return [Hash{String => String}]
+    # The full env Hash Restic will see for this repository: +RESTIC_REPOSITORY+
+    # plus resolved password and backend credentials.
     def env
       {"RESTIC_REPOSITORY" => url}
         .merge(resolved_password)
         .merge(resolved_backend_credentials)
     end
 
-    # @return [String]
+    # Redacts sensitive values so a stray +pp+/+puts+/log call never prints a
+    # credential in plaintext.
     def inspect
       "#<#{self.class} url=#{url.inspect} password=#{redacted_password.inspect} backend_credentials=#{redacted_backend_credentials.inspect}>"
     end
 
+    # Returns a Yobi::CancellableProxy wrapping this repository, exposing
+    # only its long-running methods (#backup, #restore, #check, #prune,
+    # #forget, #copy) with +token+ threaded through them, so #cancel! can
+    # stop whichever one is called:
+    #
+    #   token = Yobi::Cancellation.new
+    #   Thread.new { token.cancel! if user_clicked_cancel }
+    #   repo.with_cancellation(token).backup(source: "/data")
+    def with_cancellation(token)
+      CancellableProxy.new(self, token)
+    end
+
     private
 
+    # Set for the duration of one call by a Yobi::CancellableProxy, on the
+    # calling thread only.
+    def cancellation
+      Thread.current[:yobi_cancellation]
+    end
+
     def run_restic(argv, extra_env: {}, output: nil, &block)
-      @restic.run(argv, extra_env: env.merge(extra_env), output: output, &block)
+      @restic.run(argv, extra_env: env.merge(extra_env), output: output, cancellation: cancellation, &block)
     end
 
     def build_argv(*base)
@@ -63,14 +107,10 @@ module Yobi
       builder.to_a
     end
 
-    # Parses a command's captured output as JSON. If parsing fails, raises
-    # {Yobi::ResticCommandFailed} when `execution[:exit_code]` is `3`
-    # (Restic reported a problem alongside otherwise-JSON output), or
-    # re-raises the original +JSON::ParserError+ when it's `0`.
-    #
-    # @param execution [Hash] `{exit_code:, output:, argv:}`
-    # @return [Object] the parsed JSON
-    # @raise [Yobi::ResticCommandFailed] if parsing fails and `exit_code` is `3`
+    # Restic exit code 3 means "succeeded with warnings" and may mix a
+    # plain-text diagnostic into otherwise-JSON output. Surface that as a
+    # ResticCommandFailed instead of a bare JSON::ParserError. Exit code 0
+    # with unparseable output is a real bug; re-raise the parse error.
     def parse_json_output(execution)
       JSON.parse(execution[:output].to_s)
     rescue JSON::ParserError
@@ -78,6 +118,37 @@ module Yobi
       raise Yobi::ResticCommandFailed, execution
     end
 
+    # Same as parse_json_output but tolerant of extra lines around the
+    # JSON, from either direction. Known cases at time of writing:
+    #
+    # - Restic 0.18.0 and 0.18.1 tack prune chatter ("loading indexes...",
+    #   counts, etc.) onto the end of `restic forget --prune`'s output,
+    #   after the summary line. Fixed in 0.19.0.
+    # - Restic 0.19.0 prints a "[0:00] 100.00% ..." progress line before
+    #   `restic stats`'s JSON summary. Fixed in 0.19.1. Note that the
+    #   progress line itself starts with "[", so checking the first char
+    #   alone isn't enough - each candidate line has to actually parse.
+    # - `restic forget` on any version prints a
+    #   "Remove(snapshot/<id>) failed: <error>" line before its JSON summary
+    #   when a snapshot listed for removal couldn't actually be deleted
+    #   (e.g. an append-only backend returns 403). The summary itself is
+    #   still valid JSON, so the caller can see what did/didn't get removed.
+    #
+    # The first line that both looks like JSON ("[" or "{") and actually
+    # parses wins. If none does, falls back to parsing the whole output
+    # verbatim so an empty/malformed response still raises the right thing.
+    def parse_json_output_permissively(execution)
+      execution[:output].to_s.each_line do |line|
+        next unless line.start_with?("[", "{")
+        return JSON.parse(line)
+      rescue JSON::ParserError
+        next
+      end
+      # No parseable JSON line found. Fall back to strict parsing on the
+      # whole output so exit-code-based error dispatch takes over.
+      parse_json_output(execution)
+    end
+
     def resolved_password
       password_env(password)
     end

data/lib/yobi/restic.rb

@@ -7,87 +7,68 @@ module Yobi
   # A specific Restic binary, plus its process-level settings that apply
   # regardless of which repository is being operated on.
   # Repository-specific identity (url, credentials) lives on
-  # {Yobi::Repository} instead.
+  # Yobi::Repository instead.
   class Restic
-    # The oldest Restic version {#run}/{#run_dump}/{#run_mount} verify the
+    # The oldest Restic version #run/#run_dump/#run_mount verify the
     # installed binary meets before executing a real command. See
-    # {Yobi::UnsupportedResticVersion}.
-    MINIMUM_VERSION = Gem::Version.new("0.17.1")
+    # Yobi::UnsupportedResticVersion.
+    MINIMUM_VERSION = Gem::Version.new("0.18.0") # :nodoc:
 
-    # Env vars {#inspect} shows in the clear; anything else is redacted.
-    ALLOWED_ENV_VARS = %w[
+    # Env vars #inspect shows in the clear; anything else is redacted.
+    ALLOWED_ENV_VARS = %w[ # :nodoc:
       RESTIC_REPOSITORY RESTIC_CACHE_DIR RESTIC_COMPRESSION RESTIC_PACK_SIZE
       RESTIC_READ_CONCURRENCY RESTIC_HOST RESTIC_PROGRESS_FPS RESTIC_CACERT
       RESTIC_TLS_CLIENT_CERT RESTIC_KEY_HINT TMPDIR TMP AWS_DEFAULT_REGION
       AWS_SHARED_CREDENTIALS_FILE AZURE_ENDPOINT_SUFFIX AZURE_FORCE_CLI_CREDENTIAL
-      RCLONE_BWLIMIT
-    ].to_set.freeze
+      RCLONE_BWLIMIT].to_set.freeze
 
-    # @return [String] path to the Restic binary
-    attr_reader :restic_path
-    # @return [String, nil] `$RESTIC_CACHE_DIR`
+    # Path to the Restic binary.
+    attr_accessor :restic_path
+    # +$RESTIC_CACHE_DIR+
     attr_accessor :cache_dir
-    # @return [String, nil] `$RESTIC_COMPRESSION`
+    # +$RESTIC_COMPRESSION+
     attr_accessor :compression
-    # @return [Integer, nil] `$RESTIC_PACK_SIZE`
+    # +$RESTIC_PACK_SIZE+
     attr_accessor :pack_size
-    # @return [Integer, nil] `$RESTIC_READ_CONCURRENCY`
+    # +$RESTIC_READ_CONCURRENCY+
     attr_accessor :read_concurrency
-    # @return [String, nil] `$RESTIC_HOST`
+    # +$RESTIC_HOST+
     attr_accessor :host
-    # @return [Integer, nil] `$RESTIC_PROGRESS_FPS`
+    # +$RESTIC_PROGRESS_FPS+
     attr_accessor :progress_fps
-    # @return [String, nil] `$RESTIC_CACERT`
+    # +$RESTIC_CACERT+
     attr_accessor :cacert
-    # @return [String, nil] `$RESTIC_TLS_CLIENT_CERT`
+    # +$RESTIC_TLS_CLIENT_CERT+
     attr_accessor :tls_client_cert
-    # @return [String, nil] `$RESTIC_KEY_HINT`
+    # +$RESTIC_KEY_HINT+
     attr_accessor :key_hint
-    # @return [String, nil] `--limit-download`
+    # +--limit-download+
     attr_accessor :limit_download
-    # @return [String, nil] `--limit-upload`
+    # +--limit-upload+
     attr_accessor :limit_upload
-    # @return [String, nil] `--retry-lock`
+    # +--retry-lock+
     attr_accessor :retry_lock
-    # @return [Boolean] `--no-lock`
+    # +--no-lock+
     attr_accessor :no_lock
-    # @return [Boolean] `--no-cache`
+    # +--no-cache+
     attr_accessor :no_cache
-    # @return [Boolean] `--cleanup-cache`
+    # +--cleanup-cache+
     attr_accessor :cleanup_cache
-    # @return [Boolean] `--no-extra-verify`
+    # +--no-extra-verify+
     attr_accessor :no_extra_verify
-    # @return [String, nil] `--stuck-request-timeout`
+    # +--stuck-request-timeout+
     attr_accessor :stuck_request_timeout
-    # @return [Array<String>] `--option`, one per element
+    # +--option+, one per element.
     attr_accessor :options
-    # @return [String, nil] `--http-user-agent`
+    # +--http-user-agent+
     attr_accessor :http_user_agent
-    # @return [Boolean] `--quiet`
+    # +--quiet+
     attr_accessor :quiet
 
-    # @param restic_path [String, nil] defaults to `$RESTIC_PATH`, then `"restic"`
-    # @param env [Hash{String => String}] extra env vars, merged over the named settings below
-    # @param cache_dir [String, nil]
-    # @param compression [String, nil]
-    # @param pack_size [Integer, nil]
-    # @param read_concurrency [Integer, nil]
-    # @param host [String, nil]
-    # @param progress_fps [Integer, nil]
-    # @param cacert [String, nil]
-    # @param tls_client_cert [String, nil]
-    # @param key_hint [String, nil]
-    # @param limit_download [String, nil]
-    # @param limit_upload [String, nil]
-    # @param retry_lock [String, nil]
-    # @param no_lock [Boolean]
-    # @param no_cache [Boolean]
-    # @param cleanup_cache [Boolean]
-    # @param no_extra_verify [Boolean]
-    # @param stuck_request_timeout [String, nil]
-    # @param options [Array<String>]
-    # @param http_user_agent [String, nil]
-    # @param quiet [Boolean]
+    # Builds a Restic executor. +restic_path+ defaults to +$RESTIC_PATH+,
+    # then +"restic"+. +env:+ is extra env vars merged over the named
+    # settings below (env-var-backed accessors like +cache_dir:+,
+    # +compression:+, etc.).
     def initialize(restic_path = nil, env: {}, cache_dir: nil, compression: nil, pack_size: nil,
       read_concurrency: nil, host: nil, progress_fps: nil, cacert: nil, tls_client_cert: nil,
       key_hint: nil, limit_download: nil, limit_upload: nil, retry_lock: nil, no_lock: false,
@@ -118,10 +99,8 @@ module Yobi
     end
 
     # The env-var-backed settings above, computed fresh from their current
-    # accessor values on every call. `env:` given at construction wins over
+    # accessor values on every call. +env:+ given at construction wins over
     # any of these on key collision.
-    #
-    # @return [Hash{String => String}]
     def env
       {
         "RESTIC_CACHE_DIR" => cache_dir,
@@ -137,12 +116,9 @@ module Yobi
     end
 
     # Appends the CLI-only global flags (the ones above with no env var
-    # equivalent) to a builder. Called from both {#build_argv} and
-    # `Repository#build_argv`.
-    #
-    # @param a [Yobi::ArgvBuilder]
-    # @return [void]
-    def append_global_flags(a)
+    # equivalent) to a builder. Called from both #build_argv and
+    # Repository#build_argv.
+    def append_global_flags(a) # :nodoc:
       a.flag(:limit_download, limit_download) unless limit_download.nil?
       a.flag(:limit_upload, limit_upload) unless limit_upload.nil?
       a.flag(:retry_lock, retry_lock) unless retry_lock.nil?
@@ -156,26 +132,22 @@ module Yobi
       a.flag(:quiet) if quiet
     end
 
-    # @return [String]
+    # Redacts sensitive env values so a stray +pp+/+puts+/log call never
+    # prints a credential in plaintext.
     def inspect
       "#<#{self.class} restic_path=#{restic_path.inspect} env=#{redacted_env.inspect}>"
     end
 
-    # `restic version`: the installed binary's own version info.
-    #
-    # @return [Yobi::ResticVersion]
+    # +restic version+: the installed binary's own version info. Returns a
+    # Yobi::ResticVersion.
     def version
       execution = run(build_argv("version"), skip_version_check: true)
       Yobi::ResticVersion.new(parse_version_output(execution[:output].to_s))
     end
 
-    # `restic cache`: lists and optionally cleans local cache directories.
-    # Not repository-scoped.
-    #
-    # @param cleanup [Boolean] `--cleanup`
-    # @param max_age [String, nil] `--max-age`
-    # @param no_size [Boolean] `--no-size`
-    # @return [true]
+    # +restic cache+: lists and optionally cleans local cache directories.
+    # Not repository-scoped. +cleanup:+ triggers +--cleanup+, +max_age:+
+    # sets +--max-age+, +no_size:+ triggers +--no-size+. Returns +true+.
     def cache(cleanup: false, max_age: nil, no_size: false)
       argv = build_argv("cache") do |a|
         a.flag(:cleanup) if cleanup
@@ -186,32 +158,39 @@ module Yobi
       true
     end
 
-    # Runs argv against this Restic binary, merging extra_env with this
-    # instance's own global env.
+    # Runs +argv+ against this Restic binary, merging +extra_env:+ with
+    # this instance's own global env. +output:+ takes a caller-configured
+    # Yobi::ResticOutput (e.g. with its own +transform:+); passing one forces
+    # streaming even without a block. When a block is given, each parsed
+    # message is yielded live as the command runs.
+    #
+    # +skip_version_check:+ is used internally by #version to avoid recursing
+    # into #ensure_minimum_version!
     #
-    # @param argv [Array<String>]
-    # @param extra_env [Hash{String => String}]
-    # @param skip_version_check [Boolean] used internally by {#version} to avoid recursing into {#ensure_minimum_version!}
-    # @param output [Yobi::ResticOutput, nil] a caller-configured one (e.g. with its own `transform:`),
-    #   used instead of a plain one this creates itself; forces streaming even without a block
-    # @yieldparam message [Object] each message, live, as the command runs - transformed via
-    #   `output`'s own `transform:` if it has one, the raw parsed Hash otherwise
-    # @return [Hash] `{exit_code:, output:, argv:}` on exit code 0/3
-    # @raise [Yobi::RepositoryNotFound, Yobi::RepositoryLocked, Yobi::AuthenticationFailed, Yobi::ResticCommandFailed]
-    def run(argv, extra_env: {}, skip_version_check: false, output: nil, &block)
+    # Returns +{exit_code:, output:, argv:}+ on exit code 0 or 3.
+    # Raises Yobi::RepositoryNotFound, Yobi::RepositoryLocked,
+    # Yobi::AuthenticationFailed, Yobi::Cancelled (exit code 130, e.g. from
+    # a Yobi::Cancellation token's #cancel!), or Yobi::ResticCommandFailed
+    # otherwise.
+    def run(argv, extra_env: {}, skip_version_check: false, output: nil, cancellation: nil, &block) # :nodoc:
       ensure_minimum_version! unless skip_version_check
+      raise Yobi::Cancelled.new({exit_code: nil, output: nil, argv: argv}) if cancellation&.cancelled?
+
+      # Only forwarded when a token was actually supplied, so the common
+      # short metadata commands keep their original call shape.
+      cancel_kwarg = cancellation ? {cancellation: cancellation} : {}
       execution = if output || block
-        execute_with_streaming(argv, extra_env, output: output, &block)
+        execute_with_streaming(argv, extra_env, output: output, **cancel_kwarg, &block)
       else
-        execute(argv, extra_env)
+        execute(argv, extra_env, **cancel_kwarg)
       end
+
       self.class.dispatch(execution)
     end
 
-    # @param execution [Hash] `{exit_code:, output:, argv:}`
-    # @return [Hash] `execution`, unchanged, on exit code 0/3
-    # @raise [Yobi::RepositoryNotFound, Yobi::RepositoryLocked, Yobi::AuthenticationFailed, Yobi::ResticCommandFailed]
-    def self.dispatch(execution)
+    # Maps Restic's exit codes to Yobi's typed errors. Returns +execution+
+    # unchanged on 0/3.
+    def self.dispatch(execution) # :nodoc:
       case execution[:exit_code]
       when 0, 3
         execution
@@ -221,27 +200,23 @@ module Yobi
         raise Yobi::RepositoryLocked, execution
       when 12
         raise Yobi::AuthenticationFailed, execution
+      when 130
+        raise Yobi::Cancelled, execution
       else
         raise Yobi::ResticCommandFailed, execution
       end
     end
 
     # For commands whose success output is raw bytes with no JSON message
-    # framing (currently only `Repository#dump`). Spawns Restic with
-    # stdout wired to a pipe.
+    # framing (currently only Repository#dump). Spawns Restic with stdout
+    # wired to a pipe.
     #
-    # Without a block, returns a {Yobi::IOHandle} immediately; closing,
+    # Without a block, returns a Yobi::IOHandle immediately; closing,
     # reaping, and exit-code dispatch are the caller's own responsibility.
     # With one, yields the pipe's read end; it's always closed and the
     # process always reaped once the block returns or raises, before any
     # exit-code dispatch runs.
-    #
-    # @param argv [Array<String>]
-    # @param extra_env [Hash{String => String}]
-    # @yieldparam io [IO]
-    # @return [Yobi::IOHandle] if no block is given
-    # @return [Object] the block's own return value, otherwise
-    def run_dump(argv, extra_env: {})
+    def run_dump(argv, extra_env: {}) # :nodoc:
       ensure_minimum_version!
       output = Yobi::ResticOutput.new
       read_end, write_end = IO.pipe
@@ -266,13 +241,11 @@ module Yobi
       raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
     end
 
-    # Verifies the installed Restic binary meets {MINIMUM_VERSION}, once
-    # per instance (memoized). {#run}/{#run_dump}/{#run_mount} call this
+    # Verifies the installed Restic binary meets MINIMUM_VERSION, once per
+    # instance (memoized). #run/#run_dump/#run_mount call this
     # automatically before executing a real command; public so a caller
-    # can also call it explicitly to fail fast.
-    #
-    # @return [void]
-    # @raise [Yobi::UnsupportedResticVersion]
+    # can also call it explicitly to fail fast. Raises
+    # Yobi::UnsupportedResticVersion when too old.
     def ensure_minimum_version!
       return if defined?(@version_checked)
       @version_checked = true
@@ -288,7 +261,7 @@ module Yobi
     # An old enough Restic ignores --json for `version` entirely and
     # prints plain text instead (e.g. "restic 0.9.6 compiled with
     # go1.13.4 on linux/amd64"), which JSON.parse can't handle.
-    VERSION_LINE_PATTERN = /restic\s+(\d+\.\d+\.\d+\S*)/
+    VERSION_LINE_PATTERN = /restic\s+(\d+\.\d+\.\d+\S*)/ # :nodoc:
 
     def parse_version_output(raw)
       JSON.parse(raw)
@@ -318,21 +291,25 @@ module Yobi
       end
     end
 
-    def execute(argv, extra_env)
+    def execute(argv, extra_env, cancellation: nil)
       output = Yobi::ResticOutput.new
       pid = Process.spawn(env.merge(extra_env), restic_path, *argv, out: output.file, err: output.file)
+      cancellation&.attach(pid)
       _, status = Process.wait2(pid)
       {exit_code: status.exitstatus, output: output, argv: argv}
     rescue Errno::ENOENT
       output.file.close
       raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
+    ensure
+      cancellation&.detach
     end
 
-    def execute_with_streaming(argv, extra_env, output: nil)
+    def execute_with_streaming(argv, extra_env, output: nil, cancellation: nil)
       output ||= Yobi::ResticOutput.new
 
       Open3.popen3(env.merge(extra_env), restic_path, *argv) do |stdin, stdout, stderr, wait_thr|
         stdin.close
+        cancellation&.attach(wait_thr.pid)
         readers = {stdout => :stdout, stderr => :stderr}
 
         until readers.empty?
@@ -354,27 +331,29 @@ module Yobi
     rescue Errno::ENOENT
       output.file.close
       raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
+    ensure
+      cancellation&.detach
     end
   end
 
-  # The result of one {Yobi::Restic#version} call.
+  # The result of one Yobi::Restic#version call.
   class ResticVersion < Yobi::FancyHash
-    # @return [String]
+    # The Restic version string, e.g. +"0.19.1"+.
     def version
       self["version"]
     end
 
-    # @return [String, nil]
+    # The Go compiler version Restic was built with, if reported.
     def go_version
       self["go_version"]
     end
 
-    # @return [String, nil]
+    # The OS Restic was built for, if reported.
     def go_os
       self["go_os"]
     end
 
-    # @return [String, nil]
+    # The CPU architecture Restic was built for, if reported.
     def go_arch
       self["go_arch"]
     end