yobi 0.3.0 → 1.0.0

data/lib/yobi/repository/rewrite.rb

@@ -2,29 +2,24 @@
 
 module Yobi
   class Repository
-    # `restic rewrite`: creates new snapshots from existing ones with
+    # +restic rewrite+: creates new snapshots from existing ones with
     # exclude/include filters applied, or metadata changed. With
-    # `snapshot_ids:` and the other filters all left at their defaults,
+    # +snapshot_ids:+ and the other filters all left at their defaults,
     # rewrites every snapshot in the repository.
     #
-    # @param snapshot_ids [Array<String>, String] snapshots to rewrite; all of them if empty
-    # @param hosts [Array<String>, String] filter by hostname(s)
-    # @param tags [Array<String>, String] filter by tag(s)
-    # @param paths [Array<String>, String] filter by originally backed-up path(s)
-    # @param excludes [Array<String>, String] exclude files matching these glob patterns
-    # @param exclude_files [Array<String>, String] path(s) to file(s) listing exclude patterns
-    # @param iexcludes [Array<String>, String] like `excludes`, case-insensitive
-    # @param iexclude_files [Array<String>, String] like `exclude_files`, case-insensitive
-    # @param includes [Array<String>, String] only include files matching these glob patterns
-    # @param include_files [Array<String>, String] path(s) to file(s) listing include patterns
-    # @param iincludes [Array<String>, String] like `includes`, case-insensitive
-    # @param iinclude_files [Array<String>, String] like `include_files`, case-insensitive
-    # @param dry_run [Boolean] report what would happen without doing it
-    # @param forget [Boolean] remove the original snapshots afterward, instead of tagging the new ones `"rewrite"` and keeping both
-    # @param new_host [String, nil] change the recorded hostname
-    # @param new_time [String, nil] change the recorded timestamp
-    # @param snapshot_summary [Boolean] regenerate the snapshot summary
-    # @return [true]
+    # +snapshot_ids:+ names the snapshots to rewrite, or all of them if
+    # empty. +hosts:+, +tags:+, and +paths:+ each accept a single value or
+    # an Array to filter by.
+    #
+    # +excludes:+, +exclude_files:+, +includes:+, +include_files:+ and
+    # their +i+-prefixed (case-insensitive) variants each accept a single
+    # value or an Array.
+    #
+    # +dry_run:+ reports what would happen without doing it. +forget:+
+    # removes the original snapshots afterward, instead of tagging the new
+    # ones +"rewrite"+ and keeping both. +new_host:+ / +new_time:+ change
+    # the recorded hostname/timestamp. +snapshot_summary:+ regenerates the
+    # snapshot summary.
     def rewrite(snapshot_ids: [], hosts: [], tags: [], paths: [], excludes: [], exclude_files: [],
       iexcludes: [], iexclude_files: [], includes: [], include_files: [], iincludes: [], iinclude_files: [],
       dry_run: false, forget: false, new_host: nil, new_time: nil, snapshot_summary: false)

data/lib/yobi/repository/snapshots.rb

@@ -4,15 +4,14 @@ require "json"
 
 module Yobi
   class Repository
-    # `restic snapshots`: lists snapshots, optionally filtered.
+    # +restic snapshots+: lists snapshots, optionally filtered.
     #
-    # @param tags [Array<String>, String] filter by tag(s)
-    # @param hosts [Array<String>, String] filter by hostname(s)
-    # @param paths [Array<String>, String] filter by originally backed-up path(s)
-    # @param compact [Boolean] compact the printed listing
-    # @param group_by [String, nil] group results, e.g. `"host"`
-    # @param latest [Integer, nil] limit to the N most recent per group
-    # @return [Array<Yobi::Snapshot>]
+    # +tags:+, +hosts:+, and +paths:+ each accept a single value or an
+    # Array to filter by. +compact:+ compacts the printed listing.
+    # +group_by:+ (e.g. +"host"+) groups results. +latest:+ limits to the
+    # N most recent per group.
+    #
+    # Returns an Array of Yobi::Snapshot.
     def snapshots(tags: [], hosts: [], paths: [], compact: false, group_by: nil, latest: nil)
       argv = build_argv("snapshots") do |a|
         a.repeat_flag(:tag, tags)
@@ -23,7 +22,7 @@ module Yobi
         a.flag(:latest, latest) unless latest.nil?
       end
       execution = run_restic(argv)
-      JSON.parse(execution[:output].to_s).map { |raw| Snapshot.new(raw) }
+      parse_json_output(execution).map { |raw| Snapshot.new(raw) }
     end
   end
 end

data/lib/yobi/repository/stats.rb

@@ -1,24 +1,24 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Yobi
   class Repository
-    # @private
-    STATS_MODES = %w[restore-size files-by-contents blobs-per-file raw-data].each_with_object({}) do |value, hash|
+    STATS_MODES = %w[restore-size files-by-contents blobs-per-file raw-data].each_with_object({}) do |value, hash| # :nodoc:
       hash[value] = value
       hash[value.tr("-", "_").to_sym] = value
     end.freeze
 
-    # `restic stats`: accumulates statistics about the repository's data.
+    # +restic stats+: accumulates statistics about the repository's data.
+    #
+    # +snapshot_ids:+ restricts to specific snapshot IDs, or the whole
+    # repository if empty. +hosts:+, +paths:+, and +tags:+ each accept a
+    # single value or an Array to filter by.
+    #
+    # +mode:+ is +"restore-size"+ / +:restore_size+ (default),
+    # +"files-by-contents"+ / +:files_by_contents+, +"blobs-per-file"+ /
+    # +:blobs_per_file+, or +"raw-data"+ / +:raw_data+. Anything else
+    # raises ArgumentError.
     #
-    # @param snapshot_ids [Array<String>, String] restrict to these snapshot ID(s); the whole repository if empty
-    # @param hosts [Array<String>, String] filter by hostname(s)
-    # @param mode [String, Symbol, nil] `"restore-size"`/`:restore_size` (default), `"files-by-contents"`/`:files_by_contents`, `"blobs-per-file"`/`:blobs_per_file`, or `"raw-data"`/`:raw_data`
-    # @param paths [Array<String>, String] filter by originally backed-up path(s)
-    # @param tags [Array<String>, String] filter by tag(s)
-    # @return [Yobi::RepositoryStats]
-    # @raise [ArgumentError] if `mode:` isn't one of the values listed above
+    # Returns a Yobi::RepositoryStats.
     def stats(snapshot_ids: [], hosts: [], mode: nil, paths: [], tags: [])
       argv = build_argv("stats", snapshot_ids) do |a|
         a.repeat_flag(:host, hosts)
@@ -29,49 +29,52 @@ module Yobi
         a.repeat_flag(:tag, tags)
       end
       execution = run_restic(argv)
-      RepositoryStats.new(JSON.parse(execution[:output].to_s))
+      # Permissive because Restic 0.19.0 prints a "[0:00] 100.00% ..."
+      # progress line before the JSON summary. Fixed in 0.19.1.
+      RepositoryStats.new(parse_json_output_permissively(execution))
     end
   end
 
-  # The result of one {Yobi::Repository#stats} call.
-  # https://restic.readthedocs.io/en/stable/075_scripting.html#stats
+  # The result of one Yobi::Repository#stats call. See
+  # https://restic.readthedocs.io/en/stable/075_scripting.html#stats.
   class RepositoryStats < Yobi::FancyHash
-    # @return [Integer] repository size in bytes
+    # Repository size in bytes.
     def total_size
       self["total_size"] || 0
     end
 
-    # @return [Integer] number of files backed up in the repository
+    # Number of files backed up in the repository.
     def total_file_count
       self["total_file_count"] || 0
     end
 
-    # @return [Integer] number of blobs in the repository
+    # Number of blobs in the repository.
     def total_blob_count
       self["total_blob_count"] || 0
     end
 
-    # @return [Integer] number of processed snapshots
+    # Number of processed snapshots.
     def snapshots_count
       self["snapshots_count"] || 0
     end
 
-    # @return [Integer] repository size in bytes if blobs were uncompressed
+    # Repository size in bytes if blobs were uncompressed.
     def total_uncompressed_size
       self["total_uncompressed_size"] || 0
     end
 
-    # @return [Float] factor by which the already compressed data has shrunk due to compression
+    # Factor by which the already compressed data has shrunk due to
+    # compression.
     def compression_ratio
       self["compression_ratio"] || 0.0
     end
 
-    # @return [Float] percentage of already compressed data
+    # Percentage of already compressed data.
     def compression_progress
       self["compression_progress"] || 0.0
     end
 
-    # @return [Float] overall space saving due to compression
+    # Overall space saving due to compression.
     def compression_space_saving
       self["compression_space_saving"] || 0.0
     end

data/lib/yobi/repository/tag.rb

@@ -2,18 +2,22 @@
 
 module Yobi
   class Repository
-    # `restic tag`: modifies tags on existing snapshots. Tags are part of
-    # a snapshot's content-addressed identity, so changing them produces a
+    # +restic tag+: modifies tags on existing snapshots. Tags are part of a
+    # snapshot's content-addressed identity, so changing them produces a
     # new snapshot ID for each affected snapshot.
     #
-    # @param snapshot_ids [Array<String>, String] snapshots to modify; all matching `hosts:`/`paths:` if empty
-    # @param add [Array<String>, String] tags to add, keeping existing ones
-    # @param remove [Array<String>, String] tags to remove
-    # @param set [Array<String>, String] replace all tags with exactly this set (exclusive with `add:`/`remove:`)
-    # @param tags [Array<String>, String] filter which snapshots to modify, by their current tags
-    # @param hosts [Array<String>, String] filter which snapshots to modify, by hostname
-    # @param paths [Array<String>, String] filter which snapshots to modify, by originally backed-up path
-    # @return [Yobi::TagOutcome]
+    # +snapshot_ids:+ names the snapshots to modify, or all matching
+    # +hosts:+/+paths:+ if empty.
+    #
+    # +add:+ appends tags, keeping existing ones. +remove:+ strips tags.
+    # +set:+ replaces all tags with exactly the given set (exclusive with
+    # +add:+/+remove:+).
+    #
+    # +tags:+, +hosts:+, and +paths:+ each accept a single value or an
+    # Array to filter which snapshots to modify (by current tags,
+    # hostname, or originally backed-up path respectively).
+    #
+    # Returns a Yobi::TagOutcome.
     def tag(snapshot_ids: [], add: [], remove: [], set: [], tags: [], hosts: [], paths: [])
       argv = build_argv("tag", snapshot_ids) do |a|
         a.repeat_flag(:add, add)
@@ -29,16 +33,7 @@ module Yobi
     end
   end
 
-  # Classifies a raw tag message Hash by its own message_type, wrapping it
-  # in the matching typed class. Used both as the {Yobi::ResticOutput}
-  # `transform:` for {Yobi::Repository#tag} and by {Yobi::TagOutcome}'s own
-  # post-hoc accessors.
-  #
-  # @private
-  module TagMessageWrapper
-    # @param raw [Hash]
-    # @return [Yobi::TagChange, Yobi::TagSummary, Hash]
-    #   the raw Hash itself for a message_type this version of Yobi doesn't recognize
+  module TagMessageWrapper # :nodoc:
     def self.call(raw)
       case raw["message_type"]
       when "changed" then TagChange.new(raw)
@@ -48,28 +43,26 @@ module Yobi
     end
   end
 
-  # The outcome of one {Yobi::Repository#tag} call.
+  # The outcome of one Yobi::Repository#tag call.
   class TagOutcome
-    # @return [Yobi::ResticOutput]
+    # The Yobi::ResticOutput backing this outcome.
     attr_reader :output
 
-    # @private
-    def initialize(execution)
+    def initialize(execution) # :nodoc:
       @output = execution[:output]
     end
 
-    # @return [Yobi::TagSummary] Restic's own `"summary"` fields (`"changed_snapshots"`)
+    # The Yobi::TagSummary of Restic's own +"summary"+ fields.
     def summary
       @summary ||= output.messages("summary").first || TagSummary.new({})
     end
     alias_method :report, :summary
 
-    # @return [Enumerable<Yobi::TagChange>] one per snapshot actually modified
+    # One Yobi::TagChange per snapshot actually modified.
     def changes
       @changes ||= output.messages("changed")
     end
 
-    # @return [void]
     def pretty_print(q)
       q.object_group(self) do
         q.breakable
@@ -78,25 +71,22 @@ module Yobi
     end
   end
 
-  # One `"changed"` message from a tag run. `old_snapshot_id` is now stale:
+  # One +"changed"+ message from a tag run. #old_snapshot_id is now stale:
   # tags are part of a snapshot's content-addressed identity, so changing
   # them produces a new snapshot ID.
   class TagChange < Yobi::FancyHash
-    # @return [String]
     def old_snapshot_id
       self["old_snapshot_id"]
     end
 
-    # @return [String]
     def new_snapshot_id
       self["new_snapshot_id"]
     end
   end
 
-  # The `"summary"` message from a tag run, the final result once the
-  # command finishes. Dispatched to {Yobi::TagOutcome#summary}.
+  # The +"summary"+ message from a tag run, the final result once the
+  # command finishes. Dispatched to Yobi::TagOutcome#summary.
   class TagSummary < Yobi::FancyHash
-    # @return [Integer]
     def changed_snapshots
       self["changed_snapshots"] || 0
     end

data/lib/yobi/repository/unlock.rb

@@ -2,10 +2,8 @@
 
 module Yobi
   class Repository
-    # `restic unlock`: removes stale locks left by other Restic processes.
-    #
-    # @param remove_all [Boolean] remove every lock, not just stale ones
-    # @return [true]
+    # +restic unlock+: removes stale locks left by other Restic processes.
+    # +remove_all: true+ removes every lock, not just stale ones.
     def unlock(remove_all: false)
       argv = build_argv("unlock") do |a|
         a.flag(:remove_all) if remove_all

data/lib/yobi/repository.rb

@@ -1,29 +1,54 @@
 # frozen_string_literal: true
 
+require "json"
 require "uri"
 
 module Yobi
   # One Restic repository. Every Restic subcommand that operates on a
   # repository is a method here.
   class Repository
-    # @return [String] the repository location
+    # The repository location.
     attr_reader :url
-    # @return [String, Array, Symbol, #call] the repository's encryption password, as given to {#initialize}
+    # The repository's encryption password, as given to #initialize.
     attr_reader :password
-    # @return [Hash, #call] the storage backend's own credentials, as given to {#initialize}
+    # The storage backend's own credentials, as given to #initialize.
     attr_reader :backend_credentials
 
-    # @param url [String] the repository location, e.g. `"s3:s3.amazonaws.com/bucket"`
-    # @param password [String, Array, Symbol, #call] a literal password; a
-    #   `[:command, "..."]`/`[:file, "..."]` tuple, resolved natively by Restic
-    #   itself; `:insecure_no_password`; or anything responding to `#call`
-    #   (invoked fresh immediately before every Restic invocation)
-    # @param backend_credentials [Hash{String => String}, #call] the storage
-    #   backend's own env vars (`AWS_*`/`AZURE_*`/etc.), or anything
-    #   responding to `#call` returning such a Hash
-    # @param restic [Yobi::Restic, String, nil] a `Restic` instance to share,
-    #   a bare Restic binary path, or `nil` to create a default one
-    # @raise [ArgumentError] if `password:` is `nil`, or if `password:`/`backend_credentials:`/`restic:` is an invalid shape
+    # Sets #url and re-extracts any REST credentials embedded in it.
+    def url=(value)
+      @url, @extracted_rest_credentials = extract_rest_credentials(value)
+    end
+
+    # Sets #password. Validated on assignment with the same shapes #initialize accepts.
+    def password=(value)
+      validate_password_shape!(value)
+      @password = value
+    end
+
+    # Sets #backend_credentials. Validated on assignment with the same shapes #initialize accepts.
+    def backend_credentials=(value)
+      validate_backend_credentials_shape!(value)
+      @backend_credentials = value
+    end
+
+    # Builds a repository handle. Does not touch the remote or spawn Restic.
+    #
+    # +url+ is a String repository location, e.g. <tt>"s3:s3.amazonaws.com/bucket"</tt>.
+    #
+    # +password+ is the repository's encryption password. Accepts one of:
+    # a literal String; a <tt>[:command, "..."]</tt> or <tt>[:file, "..."]</tt>
+    # tuple, resolved natively by Restic itself; <tt>:insecure_no_password</tt>;
+    # or any object responding to +#call+ (invoked fresh immediately before every
+    # Restic invocation).
+    #
+    # +backend_credentials+ is the storage backend's own env vars
+    # (+AWS_*+/+AZURE_*+/etc.) as a Hash, or a callable returning such a Hash.
+    #
+    # +restic+ is a Yobi::Restic instance to share, a bare Restic binary path,
+    # or +nil+ to create a default one.
+    #
+    # Raises ArgumentError when +password:+ is +nil+, or when +password:+,
+    # +backend_credentials:+, or +restic:+ has an invalid shape.
     def initialize(url:, password:, backend_credentials: {}, restic: nil)
       validate_password_shape!(password)
       validate_backend_credentials_shape!(backend_credentials)
@@ -34,14 +59,16 @@ module Yobi
       @restic = initialize_restic(restic)
     end
 
-    # @return [Hash{String => String}]
+    # The full env Hash Restic will see for this repository: +RESTIC_REPOSITORY+
+    # plus resolved password and backend credentials.
     def env
       {"RESTIC_REPOSITORY" => url}
         .merge(resolved_password)
         .merge(resolved_backend_credentials)
     end
 
-    # @return [String]
+    # Redacts sensitive values so a stray +pp+/+puts+/log call never prints a
+    # credential in plaintext.
     def inspect
       "#<#{self.class} url=#{url.inspect} password=#{redacted_password.inspect} backend_credentials=#{redacted_backend_credentials.inspect}>"
     end
@@ -62,6 +89,48 @@ module Yobi
       builder.to_a
     end
 
+    # Restic exit code 3 means "succeeded with warnings" and may mix a
+    # plain-text diagnostic into otherwise-JSON output. Surface that as a
+    # ResticCommandFailed instead of a bare JSON::ParserError. Exit code 0
+    # with unparseable output is a real bug; re-raise the parse error.
+    def parse_json_output(execution)
+      JSON.parse(execution[:output].to_s)
+    rescue JSON::ParserError
+      raise if execution[:exit_code].zero?
+      raise Yobi::ResticCommandFailed, execution
+    end
+
+    # Same as parse_json_output but tolerant of extra lines around the
+    # JSON, from either direction. Known cases at time of writing:
+    #
+    # - Restic 0.18.0 and 0.18.1 tack prune chatter ("loading indexes...",
+    #   counts, etc.) onto the end of `restic forget --prune`'s output,
+    #   after the summary line. Fixed in 0.19.0.
+    # - Restic 0.19.0 prints a "[0:00] 100.00% ..." progress line before
+    #   `restic stats`'s JSON summary. Fixed in 0.19.1. Note that the
+    #   progress line itself starts with "[", so checking the first char
+    #   alone isn't enough - each candidate line has to actually parse.
+    # - `restic forget` on any version prints a
+    #   "Remove(snapshot/<id>) failed: <error>" line before its JSON summary
+    #   when a snapshot listed for removal couldn't actually be deleted
+    #   (e.g. an append-only backend returns 403). The summary itself is
+    #   still valid JSON, so the caller can see what did/didn't get removed.
+    #
+    # The first line that both looks like JSON ("[" or "{") and actually
+    # parses wins. If none does, falls back to parsing the whole output
+    # verbatim so an empty/malformed response still raises the right thing.
+    def parse_json_output_permissively(execution)
+      execution[:output].to_s.each_line do |line|
+        next unless line.start_with?("[", "{")
+        return JSON.parse(line)
+      rescue JSON::ParserError
+        next
+      end
+      # No parseable JSON line found. Fall back to strict parsing on the
+      # whole output so exit-code-based error dispatch takes over.
+      parse_json_output(execution)
+    end
+
     def resolved_password
       password_env(password)
     end