yobi 0.1.0

data/lib/yobi/repository/restore.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "json"
+require "delegate"
+
+module Yobi
+  class Repository
+    # `restic restore`: extracts a snapshot's contents to a target directory.
+    #
+    # @param snapshot_id [String] also accepts `"latest"`
+    # @param target [String] directory to restore into
+    # @param excludes [Array<String>, String] exclude files matching these glob patterns
+    # @param exclude_files [Array<String>, String] path(s) to file(s) listing exclude patterns
+    # @param exclude_xattrs [Array<String>, String] exclude extended attributes matching these pattern(s)
+    # @param hosts [Array<String>, String] only relevant when `snapshot_id:` is `"latest"`
+    # @param iexcludes [Array<String>, String] like `excludes`, case-insensitive
+    # @param iexclude_files [Array<String>, String] like `exclude_files`, case-insensitive
+    # @param iincludes [Array<String>, String] like `includes`, case-insensitive
+    # @param iinclude_files [Array<String>, String] like `include_files`, case-insensitive
+    # @param includes [Array<String>, String] only restore files matching these glob patterns
+    # @param include_files [Array<String>, String] path(s) to file(s) listing include patterns
+    # @param include_xattrs [Array<String>, String] only restore extended attributes matching these pattern(s)
+    # @param paths [Array<String>, String] only relevant when `snapshot_id:` is `"latest"`
+    # @param tags [Array<String>, String] only relevant when `snapshot_id:` is `"latest"`
+    # @param delete [Boolean] delete files in `target:` not present in the snapshot
+    # @param dry_run [Boolean] report what would happen without doing it
+    # @param overwrite [String, Symbol, nil] `"always"`, `"if-changed"`, or `"if-newer"`
+    # @param ownership_by_name [Boolean] map ownership by user/group name instead of numeric ID
+    # @param sparse [Boolean] write sparse files
+    # @param verbose [Boolean] stream a {Yobi::RestoreVerboseStatus} per file to the block
+    # @param verify [Boolean] verify restored file content against the repository
+    # @yieldparam message [Yobi::RestoreStatus, Yobi::RestoreVerboseStatus]
+    # @return [Yobi::RestoreOutcome]
+    def restore(snapshot_id:, target:, excludes: [], exclude_files: [], exclude_xattrs: [], hosts: [],
+      iexcludes: [], iexclude_files: [], iincludes: [], iinclude_files: [], includes: [], include_files: [],
+      include_xattrs: [], paths: [], tags: [], delete: false, dry_run: false, overwrite: nil,
+      ownership_by_name: false, sparse: false, verbose: false, verify: false, &block)
+      argv = build_argv("restore", snapshot_id) do |a|
+        a.flag(:target, target)
+        a.repeat_flag(:exclude, excludes)
+        a.repeat_flag(:exclude_file, exclude_files)
+        a.repeat_flag(:exclude_xattr, exclude_xattrs)
+        a.repeat_flag(:host, hosts)
+        a.repeat_flag(:iexclude, iexcludes)
+        a.repeat_flag(:iexclude_file, iexclude_files)
+        a.repeat_flag(:iinclude, iincludes)
+        a.repeat_flag(:iinclude_file, iinclude_files)
+        a.repeat_flag(:include, includes)
+        a.repeat_flag(:include_file, include_files)
+        a.repeat_flag(:include_xattr, include_xattrs)
+        a.repeat_flag(:path, paths)
+        a.repeat_flag(:tag, tags)
+        a.flag(:delete) if delete
+        a.flag(:dry_run) if dry_run
+        a.flag(:overwrite, overwrite) unless overwrite.nil?
+        a.flag(:ownership_by_name) if ownership_by_name
+        a.flag(:sparse) if sparse
+        a.short_flag(:vv) if verbose
+        a.flag(:verify) if verify
+      end
+      execution = if block
+        run_restic(argv) { |raw| dispatch_restore_message(raw, &block) }
+      else
+        run_restic(argv)
+      end
+      RestoreOutcome.new(execution[:exit_code], execution[:output])
+    end
+
+    private
+
+    def dispatch_restore_message(raw)
+      case raw["message_type"]
+      when "status"
+        yield Yobi::RestoreStatus.new(raw)
+      when "verbose_status"
+        yield Yobi::RestoreVerboseStatus.new(raw)
+      end
+    end
+  end
+
+  # The outcome of one {Yobi::Repository#restore} call.
+  class RestoreOutcome < Struct.new(:exit_code, :output)
+    # @return [Hash] Restic's own `"summary"` fields (`"total_files"`, `"files_restored"`, ...)
+    def report
+      @report ||= summary_hash
+    end
+
+    private
+
+    def summary_hash
+      line = output.last_line
+      hash = line && JSON.parse(line)
+      if hash && hash["message_type"] == "summary"
+        hash
+      else
+        find_summary || {}
+      end
+    end
+
+    def find_summary
+      output.each_line do |line|
+        next if line.strip.empty?
+
+        hash = JSON.parse(line)
+        return hash if hash["message_type"] == "summary"
+      end
+
+      nil
+    end
+  end
+
+  # One `"status"` message from a live restore run.
+  # https://restic.readthedocs.io/en/stable/075_scripting.html#restore
+  class RestoreStatus < SimpleDelegator
+    # @return [Float, nil]
+    def percent_done
+      self["percent_done"]
+    end
+
+    # @return [Integer, nil]
+    def total_files
+      self["total_files"]
+    end
+
+    # @return [Integer, nil]
+    def files_restored
+      self["files_restored"]
+    end
+
+    # @return [Integer, nil]
+    def total_bytes
+      self["total_bytes"]
+    end
+
+    # @return [Integer, nil]
+    def bytes_restored
+      self["bytes_restored"]
+    end
+  end
+
+  # One `"verbose_status"` message from a restore run, one per file. Only
+  # emitted when `verbose: true` is passed to {Yobi::Repository#restore}.
+  class RestoreVerboseStatus < SimpleDelegator
+    # @return [String]
+    def action
+      self["action"]
+    end
+
+    # @return [String]
+    def item
+      self["item"]
+    end
+
+    # @return [Integer]
+    def size
+      self["size"]
+    end
+  end
+end

data/lib/yobi/repository/rewrite.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Yobi
+  class Repository
+    # `restic rewrite`: creates new snapshots from existing ones with
+    # exclude/include filters applied, or metadata changed. With
+    # `snapshot_ids:` and the other filters all left at their defaults,
+    # rewrites every snapshot in the repository.
+    #
+    # @param snapshot_ids [Array<String>, String] snapshots to rewrite; all of them if empty
+    # @param hosts [Array<String>, String] filter by hostname(s)
+    # @param tags [Array<String>, String] filter by tag(s)
+    # @param paths [Array<String>, String] filter by originally backed-up path(s)
+    # @param excludes [Array<String>, String] exclude files matching these glob patterns
+    # @param exclude_files [Array<String>, String] path(s) to file(s) listing exclude patterns
+    # @param iexcludes [Array<String>, String] like `excludes`, case-insensitive
+    # @param iexclude_files [Array<String>, String] like `exclude_files`, case-insensitive
+    # @param includes [Array<String>, String] only include files matching these glob patterns
+    # @param include_files [Array<String>, String] path(s) to file(s) listing include patterns
+    # @param iincludes [Array<String>, String] like `includes`, case-insensitive
+    # @param iinclude_files [Array<String>, String] like `include_files`, case-insensitive
+    # @param dry_run [Boolean] report what would happen without doing it
+    # @param forget [Boolean] remove the original snapshots afterward, instead of tagging the new ones `"rewrite"` and keeping both
+    # @param new_host [String, nil] change the recorded hostname
+    # @param new_time [String, nil] change the recorded timestamp
+    # @param snapshot_summary [Boolean] regenerate the snapshot summary
+    # @return [true]
+    def rewrite(snapshot_ids: [], hosts: [], tags: [], paths: [], excludes: [], exclude_files: [],
+      iexcludes: [], iexclude_files: [], includes: [], include_files: [], iincludes: [], iinclude_files: [],
+      dry_run: false, forget: false, new_host: nil, new_time: nil, snapshot_summary: false)
+      argv = build_argv("rewrite", snapshot_ids) do |a|
+        a.repeat_flag(:host, hosts)
+        a.repeat_flag(:tag, tags)
+        a.repeat_flag(:path, paths)
+        a.repeat_flag(:exclude, excludes)
+        a.repeat_flag(:exclude_file, exclude_files)
+        a.repeat_flag(:iexclude, iexcludes)
+        a.repeat_flag(:iexclude_file, iexclude_files)
+        a.repeat_flag(:include, includes)
+        a.repeat_flag(:include_file, include_files)
+        a.repeat_flag(:iinclude, iincludes)
+        a.repeat_flag(:iinclude_file, iinclude_files)
+        a.flag(:dry_run) if dry_run
+        a.flag(:forget) if forget
+        a.flag(:new_host, new_host) unless new_host.nil?
+        a.flag(:new_time, new_time) unless new_time.nil?
+        a.flag(:snapshot_summary) if snapshot_summary
+      end
+      run_restic(argv)
+      true
+    end
+  end
+end

data/lib/yobi/repository/snapshots.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Yobi
+  class Repository
+    # `restic snapshots`: lists snapshots, optionally filtered.
+    #
+    # @param tags [Array<String>, String] filter by tag(s)
+    # @param hosts [Array<String>, String] filter by hostname(s)
+    # @param paths [Array<String>, String] filter by originally backed-up path(s)
+    # @param compact [Boolean] compact the printed listing
+    # @param group_by [String, nil] group results, e.g. `"host"`
+    # @param latest [Integer, nil] limit to the N most recent per group
+    # @return [Enumerable<Yobi::Snapshot>]
+    def snapshots(tags: [], hosts: [], paths: [], compact: false, group_by: nil, latest: nil)
+      argv = build_argv("snapshots") do |a|
+        a.repeat_flag(:tag, tags)
+        a.repeat_flag(:host, hosts)
+        a.repeat_flag(:path, paths)
+        a.flag(:compact) if compact
+        a.flag(:group_by, group_by) unless group_by.nil?
+        a.flag(:latest, latest) unless latest.nil?
+      end
+      execution = run_restic(argv)
+      Snapshots.new(execution[:output])
+    end
+  end
+
+  # Enumerable over every {Yobi::Snapshot} in one {Yobi::Repository#snapshots} call.
+  class Snapshots
+    include Enumerable
+
+    # @return [Yobi::ResticOutput]
+    attr_reader :output
+
+    # @param output [Yobi::ResticOutput]
+    def initialize(output)
+      @output = output
+    end
+
+    # @yieldparam snapshot [Yobi::Snapshot]
+    # @return [Enumerator] if no block is given
+    def each(&block)
+      return enum_for(:each) unless block_given?
+
+      entries.each(&block)
+    end
+
+    private
+
+    def entries
+      @entries ||= JSON.parse(output.to_s).map { |raw| Yobi::Snapshot.new(raw) }
+    end
+  end
+end

data/lib/yobi/repository/stats.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Yobi
+  class Repository
+    # Maps every valid `mode:` value for {#stats}, as both String and Symbol,
+    # to the String Restic itself expects.
+    STATS_MODES = %w[restore-size files-by-contents blobs-per-file raw-data].each_with_object({}) do |value, hash|
+      hash[value] = value
+      hash[value.tr("-", "_").to_sym] = value
+    end.freeze
+
+    # `restic stats`: accumulates statistics about the repository's data.
+    #
+    # @param snapshot_ids [Array<String>, String] restrict to these snapshot ID(s); the whole repository if empty
+    # @param hosts [Array<String>, String] filter by hostname(s)
+    # @param mode [String, Symbol, nil] `"restore-size"`/`:restore_size` (default), `"files-by-contents"`/`:files_by_contents`, `"blobs-per-file"`/`:blobs_per_file`, or `"raw-data"`/`:raw_data`
+    # @param paths [Array<String>, String] filter by originally backed-up path(s)
+    # @param tags [Array<String>, String] filter by tag(s)
+    # @return [Hash] `"total_size"`, `"total_file_count"`, `"snapshots_count"`, ...
+    # @raise [ArgumentError] if `mode:` isn't one of the values listed above
+    def stats(snapshot_ids: [], hosts: [], mode: nil, paths: [], tags: [])
+      argv = build_argv("stats", snapshot_ids) do |a|
+        a.repeat_flag(:host, hosts)
+        unless mode.nil?
+          a.flag(:mode, STATS_MODES.fetch(mode) { raise ArgumentError, "invalid mode: #{mode.inspect}" })
+        end
+        a.repeat_flag(:path, paths)
+        a.repeat_flag(:tag, tags)
+      end
+      execution = run_restic(argv)
+      JSON.parse(execution[:output].to_s)
+    end
+  end
+end

data/lib/yobi/repository/tag.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "json"
+require "delegate"
+
+module Yobi
+  class Repository
+    # `restic tag`: modifies tags on existing snapshots. Tags are part of
+    # a snapshot's content-addressed identity, so changing them produces a
+    # new snapshot ID for each affected snapshot.
+    #
+    # @param snapshot_ids [Array<String>, String] snapshots to modify; all matching `hosts:`/`paths:` if empty
+    # @param add [Array<String>, String] tags to add, keeping existing ones
+    # @param remove [Array<String>, String] tags to remove
+    # @param set [Array<String>, String] replace all tags with exactly this set (exclusive with `add:`/`remove:`)
+    # @param tags [Array<String>, String] filter which snapshots to modify, by their current tags
+    # @param hosts [Array<String>, String] filter which snapshots to modify, by hostname
+    # @param paths [Array<String>, String] filter which snapshots to modify, by originally backed-up path
+    # @return [Yobi::TagOutcome]
+    def tag(snapshot_ids: [], add: [], remove: [], set: [], tags: [], hosts: [], paths: [])
+      argv = build_argv("tag", snapshot_ids) do |a|
+        a.repeat_flag(:add, add)
+        a.repeat_flag(:remove, remove)
+        a.repeat_flag(:set, set)
+        a.repeat_flag(:tag, tags)
+        a.repeat_flag(:host, hosts)
+        a.repeat_flag(:path, paths)
+      end
+      execution = run_restic(argv)
+      TagOutcome.new(execution[:exit_code], execution[:output])
+    end
+  end
+
+  # The outcome of one {Yobi::Repository#tag} call.
+  class TagOutcome < Struct.new(:exit_code, :output)
+    # @return [Hash] Restic's own `"summary"` fields (`"changed_snapshots"`)
+    def report
+      @report ||= summary_hash
+    end
+
+    # @return [Array<Yobi::TagChange>] one per snapshot actually modified
+    def changes
+      @changes ||= output.each_line.filter_map do |line|
+        next if line.strip.empty?
+
+        hash = JSON.parse(line)
+        Yobi::TagChange.new(hash) if hash["message_type"] == "changed"
+      end
+    end
+
+    private
+
+    def summary_hash
+      line = output.last_line
+      hash = line && JSON.parse(line)
+      if hash && hash["message_type"] == "summary"
+        hash
+      else
+        find_summary || {}
+      end
+    end
+
+    def find_summary
+      output.each_line do |line|
+        next if line.strip.empty?
+
+        hash = JSON.parse(line)
+        return hash if hash["message_type"] == "summary"
+      end
+
+      nil
+    end
+  end
+
+  # One `"changed"` message from a tag run. `old_snapshot_id` is now stale:
+  # tags are part of a snapshot's content-addressed identity, so changing
+  # them produces a new snapshot ID.
+  class TagChange < SimpleDelegator
+    # @return [String]
+    def old_snapshot_id
+      self["old_snapshot_id"]
+    end
+
+    # @return [String]
+    def new_snapshot_id
+      self["new_snapshot_id"]
+    end
+  end
+end

data/lib/yobi/repository/unlock.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Yobi
+  class Repository
+    # `restic unlock`: removes stale locks left by other Restic processes.
+    #
+    # @param remove_all [Boolean] remove every lock, not just stale ones
+    # @return [true]
+    def unlock(remove_all: false)
+      argv = build_argv("unlock") do |a|
+        a.flag(:remove_all) if remove_all
+      end
+      run_restic(argv)
+      true
+    end
+  end
+end

data/lib/yobi/repository.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "uri"
+require "json"
+
+module Yobi
+  # One Restic repository. Every Restic subcommand that operates on a
+  # repository is a method here.
+  class Repository
+    # @return [String] the repository location
+    attr_reader :url
+    # @return [String, Array, Symbol, #call, nil] the repository's encryption password, as given to {#initialize}
+    attr_reader :password
+    # @return [Hash, #call] the storage backend's own credentials, as given to {#initialize}
+    attr_reader :backend_credentials
+
+    # @param url [String] the repository location, e.g. `"s3:s3.amazonaws.com/bucket"`
+    # @param password [String, Array, Symbol, #call, nil] a literal password; a
+    #   `[:command, "..."]`/`[:file, "..."]` tuple, resolved natively by Restic
+    #   itself; `:insecure_no_password`; or anything responding to `#call`
+    #   (invoked fresh immediately before every Restic invocation)
+    # @param backend_credentials [Hash{String => String}, #call] the storage
+    #   backend's own env vars (`AWS_*`/`AZURE_*`/etc.), or anything
+    #   responding to `#call` returning such a Hash
+    # @param restic [Yobi::Restic, String, nil] a `Restic` instance to share,
+    #   a bare Restic binary path, or `nil` to create a default one
+    def initialize(url:, password: nil, backend_credentials: {}, restic: nil)
+      @url, @extracted_rest_credentials = extract_rest_credentials(url)
+      @password = password
+      @backend_credentials = backend_credentials
+      @restic = initialize_restic(restic)
+    end
+
+    # @return [Hash{String => String}]
+    def env
+      {"RESTIC_REPOSITORY" => url}
+        .merge(resolved_password)
+        .merge(resolved_backend_credentials)
+    end
+
+    # @return [String]
+    def inspect
+      "#<#{self.class} url=#{url.inspect} password=#{redacted_password.inspect} backend_credentials=#{redacted_backend_credentials.inspect}>"
+    end
+
+    # `restic init`: creates the repository at {#url}.
+    #
+    # @param copy_chunker_params [Boolean] copy chunker parameters from `from_repo:`/`from_repository_file:`
+    # @param from_insecure_no_password [Boolean]
+    # @param from_key_hint [String, nil]
+    # @param from_password_command [String, nil]
+    # @param from_password_file [String, nil]
+    # @param from_repo [String, nil]
+    # @param from_repository_file [String, nil]
+    # @param repository_version [String, nil]
+    # @return [Hash] Restic's own `"initialized"` message
+    def init(copy_chunker_params: false, from_insecure_no_password: false, from_key_hint: nil,
+      from_password_command: nil, from_password_file: nil, from_repo: nil, from_repository_file: nil,
+      repository_version: nil)
+      argv = build_argv("init") do |a|
+        a.flag(:copy_chunker_params) if copy_chunker_params
+        a.flag(:from_insecure_no_password) if from_insecure_no_password
+        a.flag(:from_key_hint, from_key_hint) unless from_key_hint.nil?
+        a.flag(:from_password_command, from_password_command) unless from_password_command.nil?
+        a.flag(:from_password_file, from_password_file) unless from_password_file.nil?
+        a.flag(:from_repo, from_repo) unless from_repo.nil?
+        a.flag(:from_repository_file, from_repository_file) unless from_repository_file.nil?
+        a.flag(:repository_version, repository_version) unless repository_version.nil?
+      end
+      execution = run_restic(argv)
+      JSON.parse(execution[:output].to_s)
+    end
+
+    # `restic cat config`: the repository's own config document.
+    #
+    # @return [Hash]
+    # @raise [Yobi::RepositoryNotFound, Yobi::AuthenticationFailed]
+    def config
+      execution = run_restic(build_argv("cat", "config"))
+      JSON.parse(execution[:output].to_s)
+    end
+
+    private
+
+    def run_restic(argv, extra_env: {}, &block)
+      @restic.run(argv, extra_env: env.merge(extra_env), &block)
+    end
+
+    def build_argv(*base)
+      builder = ArgvBuilder.new
+      builder.append(*base)
+      builder.flag(:insecure_no_password) if password == :insecure_no_password
+      builder.flag(:json)
+      @restic.append_global_flags(builder)
+      yield builder if block_given?
+      builder.to_a
+    end
+
+    def resolved_password
+      password_env(password)
+    end
+
+    def password_env(value, base = "RESTIC_PASSWORD")
+      case value
+      in nil | :insecure_no_password
+        {}
+      in String => string
+        {base => string}
+      in [:command, String => command]
+        {"#{base}_COMMAND" => command}
+      in [:file, String => file]
+        {"#{base}_FILE" => file}
+      in callable if callable.respond_to?(:call)
+        {base => callable.call.to_s}
+      end
+    end
+
+    def resolved_backend_credentials
+      @extracted_rest_credentials.merge(resolved_backend_credentials_value)
+    end
+
+    def resolved_backend_credentials_value
+      case backend_credentials
+      in Hash => value
+        value
+      in callable if callable.respond_to?(:call)
+        callable.call
+      end
+    end
+
+    def redacted_password
+      case password
+      in nil | :insecure_no_password
+        password
+      in String | [:command, String] | [:file, String]
+        "[FILTERED]"
+      in callable if callable.respond_to?(:call)
+        "[RESOLVER]"
+      end
+    end
+
+    def redacted_backend_credentials
+      case backend_credentials
+      in Hash => value
+        value.each_with_object({}) do |(key, val), filtered|
+          filtered[key] = if Yobi::Restic::ALLOWED_ENV_VARS.include?(key)
+            val
+          else
+            "[FILTERED]"
+          end
+        end
+      in callable if callable.respond_to?(:call)
+        "[RESOLVER]"
+      end
+    end
+
+    def extract_rest_credentials(url)
+      return [url, {}] unless url.start_with?("rest:")
+
+      uri = URI.parse(url.delete_prefix("rest:"))
+      return [url, {}] unless uri.password || uri.user
+
+      extracted = {"RESTIC_REST_USERNAME" => uri.user, "RESTIC_REST_PASSWORD" => uri.password}.compact
+      uri.user = nil
+      uri.password = nil
+      ["rest:#{uri}", extracted]
+    rescue URI::InvalidURIError
+      [url, {}]
+    end
+
+    def initialize_restic(restic_or_restic_path)
+      case restic_or_restic_path
+      in Yobi::Restic
+        restic_or_restic_path
+      in String | nil
+        Yobi::Restic.new(restic_or_restic_path)
+      end
+    end
+  end
+end