yobi 0.1.0 → 0.2.0

data/lib/yobi/repository/ls.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-require "json"
 require "time"
-require "delegate"
 
 module Yobi
   class Repository
@@ -32,33 +30,67 @@ module Yobi
         a.repeat_flag(:tag, tags)
         a.end_of_options.append(dirs)
       end
-      execution = run_restic(argv)
-      LsOutcome.new(execution[:exit_code], execution[:output])
+      output = Yobi::ResticOutput.new(transform: Yobi::LsMessageWrapper)
+      execution = run_restic(argv, output: output)
+      LsOutcome.new(execution)
+    end
+  end
+
+  # Classifies a raw ls message Hash by its own message_type, wrapping it
+  # in the matching typed class. Used both as the {Yobi::ResticOutput}
+  # `transform:` for {Yobi::Repository#ls} and by {Yobi::LsOutcome}'s own
+  # post-hoc accessors.
+  #
+  # @private
+  module LsMessageWrapper
+    # @param raw [Hash]
+    # @return [Yobi::Snapshot, Yobi::Node, Hash]
+    #   the raw Hash itself for a message_type this version of Yobi doesn't recognize
+    def self.call(raw)
+      case raw["message_type"]
+      when "snapshot" then Snapshot.new(raw)
+      when "node" then Node.new(raw)
+      else raw
+      end
     end
   end
 
   # The outcome of one {Yobi::Repository#ls} call.
-  class LsOutcome < Struct.new(:exit_code, :output)
+  class LsOutcome
+    # @return [Yobi::ResticOutput]
+    attr_reader :output
+
+    # @private
+    def initialize(execution)
+      @output = execution[:output]
+    end
+
     # @return [Yobi::Snapshot] the resolved snapshot, useful when `snapshot_id:` was `"latest"`
     def snapshot
-      @snapshot ||= Yobi::Snapshot.new(JSON.parse(snapshot_line))
+      @snapshot ||= output.messages("snapshot").first
     end
 
-    # @return [Enumerable<Yobi::LsEntry>]
-    def entries
-      @entries ||= LsEntries.new(output)
+    # @return [Enumerable<Yobi::Node>]
+    def nodes
+      @nodes ||= output.messages("node")
     end
+    alias_method :entries, :nodes
 
-    private
-
-    def snapshot_line
-      offset = output.index["snapshot"].first
-      output.read_line_at(offset)
+    # @return [void]
+    def pretty_print(q)
+      q.object_group(self) do
+        q.breakable
+        q.pp snapshot
+        q.breakable
+        q.pp nodes
+      end
     end
   end
 
-  # One file/directory entry from a {Yobi::Repository#ls} call.
-  class LsEntry < SimpleDelegator
+  # One file/directory entry from a {Yobi::Repository#ls} call. Restic's
+  # own term for this is a "node" - the same structure it uses internally
+  # for every entry in a snapshot's tree, not just ones `ls` happens to show.
+  class Node < Yobi::FancyHash
     # @return [String]
     def name
       self["name"]
@@ -114,25 +146,4 @@ module Yobi
       @ctime ||= Time.parse(self["ctime"])
     end
   end
-
-  # Enumerable over every {Yobi::LsEntry} in a run.
-  class LsEntries
-    include Enumerable
-
-    # @param output [Yobi::ResticOutput]
-    def initialize(output)
-      @output = output
-    end
-
-    # @yieldparam entry [Yobi::LsEntry]
-    # @return [Enumerator] if no block is given
-    def each
-      return enum_for(:each) unless block_given?
-
-      @output.index["node"].each do |offset|
-        line = @output.read_line_at(offset)
-        yield LsEntry.new(JSON.parse(line))
-      end
-    end
-  end
 end

data/lib/yobi/repository/mount.rb

@@ -1,16 +1,15 @@
 # frozen_string_literal: true
 
 require "open3"
-require "tempfile"
 
 module Yobi
   class Repository
     # `restic mount`: serves this repository as a read-only FUSE
     # filesystem at `mountpoint`, which must already exist.
     #
-    # Without a block, returns a {Yobi::Mount} once Restic reports itself
-    # ready; call `#stop` yourself once done. With one, yields the `Mount`
-    # and stops it automatically once the block returns or raises.
+    # Without a block, returns a {Yobi::MountHandle} once Restic reports
+    # itself ready; call `#stop` yourself once done. With one, yields the
+    # `MountHandle` and stops it automatically once the block returns or raises.
     #
     # @param mountpoint [String] must already exist
     # @param hosts [Array<String>, String] restrict which snapshots appear under `snapshots/`
@@ -22,8 +21,8 @@ module Yobi
     # @param path_templates [Array<String>, String] directory naming scheme(s) under `snapshots/`
     # @param time_template [String, nil] directory naming scheme for time-based paths
     # @param ready_timeout [Numeric] seconds to wait for Restic's readiness message before raising
-    # @yieldparam mount [Yobi::Mount]
-    # @return [Yobi::Mount] if no block is given
+    # @yieldparam mount [Yobi::MountHandle]
+    # @return [Yobi::MountHandle] if no block is given
     # @return [Object] the block's own return value, otherwise
     # @raise [Yobi::MountTimeout]
     def mount(mountpoint:, hosts: [], paths: [], tags: [], allow_other: false,
@@ -59,7 +58,7 @@ module Yobi
   end
 
   class Restic
-    # Restic's own stdout line signaling a mount is ready to serve.
+    # @private
     READY_LINE = "Now serving the repository at"
 
     # For {Yobi::Repository#mount}. Spawns Restic and waits for its
@@ -70,43 +69,42 @@ module Yobi
     # @param mountpoint [String]
     # @param extra_env [Hash{String => String}]
     # @param ready_timeout [Numeric]
-    # @return [Yobi::Mount]
+    # @return [Yobi::MountHandle]
     # @raise [Yobi::MountTimeout]
     def run_mount(argv, mountpoint:, extra_env: {}, ready_timeout: 10)
       ensure_minimum_version!
-      file = Tempfile.new("yobi-restic-output")
-      file.unlink
+      output = Yobi::ResticOutput.new
 
-      stdin, output, wait_thr = Open3.popen2e(env.merge(extra_env), restic_path, *argv)
+      stdin, pipe, wait_thr = Open3.popen2e(env.merge(extra_env), restic_path, *argv)
       stdin.close
 
-      case wait_for_ready(output, file, mountpoint, ready_timeout)
+      case wait_for_ready(pipe, output.file, mountpoint, ready_timeout)
       when :ready
-        Yobi::Mount.new(wait_thr: wait_thr, mountpoint: mountpoint, output: output, output_file: file, argv: argv)
+        Yobi::MountHandle.new(wait_thr: wait_thr, mountpoint: mountpoint, pipe: pipe, output: output, argv: argv)
       when :timeout
         Process.kill("INT", wait_thr.pid)
         wait_thr.value
         raise Yobi::MountTimeout.new(argv: argv, timeout: ready_timeout)
       when :exited
         status = wait_thr.value
-        self.class.dispatch(exit_code: status.exitstatus, output: Yobi::ResticOutput.new(file), argv: argv)
+        self.class.dispatch(exit_code: status.exitstatus, output: output, argv: argv)
       end
     rescue Errno::ENOENT
-      file&.close
+      output&.file&.close
       raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
     end
 
     private
 
-    def wait_for_ready(output, file, mountpoint, timeout)
+    def wait_for_ready(pipe, file, mountpoint, timeout)
       deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
 
       loop do
         remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
         return :timeout if remaining <= 0
-        return :timeout if IO.select([output], nil, nil, remaining).nil?
+        return :timeout if IO.select([pipe], nil, nil, remaining).nil?
 
-        line = output.gets
+        line = pipe.gets
         return :exited if line.nil?
 
         file.write(line)

data/lib/yobi/repository/restore.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "delegate"
-
 module Yobi
   class Repository
     # `restic restore`: extracts a snapshot's contents to a target directory.
@@ -58,60 +55,65 @@ module Yobi
         a.short_flag(:vv) if verbose
         a.flag(:verify) if verify
       end
-      execution = if block
-        run_restic(argv) { |raw| dispatch_restore_message(raw, &block) }
-      else
-        run_restic(argv)
-      end
-      RestoreOutcome.new(execution[:exit_code], execution[:output])
-    end
 
-    private
+      output = Yobi::ResticOutput.new(transform: Yobi::RestoreMessageWrapper)
+      # Restic's own restore summary isn't meant for the live block - only
+      # {RestoreOutcome#summary}'s post-hoc access sees it - but it shares
+      # the same @transform (needed there since it isn't a status/verbose_status
+      # message), so it has to be filtered back out here instead.
+      live_block = block && proc { |message| block.call(message) if message.is_a?(RestoreStatus) || message.is_a?(RestoreVerboseStatus) }
+      execution = run_restic(argv, output: output, &live_block)
+      RestoreOutcome.new(execution)
+    end
+  end
 
-    def dispatch_restore_message(raw)
+  # Classifies a raw restore message Hash by its own message_type, wrapping
+  # it in the matching typed class. Used both by {Yobi::Repository#restore}
+  # (as the {Yobi::ResticOutput} `transform:` for a live streaming run) and
+  # by {Yobi::RestoreOutcome}'s own post-hoc accessors.
+  #
+  # @private
+  module RestoreMessageWrapper
+    # @param raw [Hash]
+    # @return [Yobi::RestoreStatus, Yobi::RestoreVerboseStatus, Hash]
+    #   the raw Hash itself for a message_type this version of Yobi doesn't recognize
+    def self.call(raw)
       case raw["message_type"]
-      when "status"
-        yield Yobi::RestoreStatus.new(raw)
-      when "verbose_status"
-        yield Yobi::RestoreVerboseStatus.new(raw)
+      when "status" then RestoreStatus.new(raw)
+      when "verbose_status" then RestoreVerboseStatus.new(raw)
+      else raw
       end
     end
   end
 
   # The outcome of one {Yobi::Repository#restore} call.
-  class RestoreOutcome < Struct.new(:exit_code, :output)
-    # @return [Hash] Restic's own `"summary"` fields (`"total_files"`, `"files_restored"`, ...)
-    def report
-      @report ||= summary_hash
-    end
-
-    private
+  class RestoreOutcome
+    # @return [Yobi::ResticOutput]
+    attr_reader :output
 
-    def summary_hash
-      line = output.last_line
-      hash = line && JSON.parse(line)
-      if hash && hash["message_type"] == "summary"
-        hash
-      else
-        find_summary || {}
-      end
+    # @private
+    def initialize(execution)
+      @output = execution[:output]
     end
 
-    def find_summary
-      output.each_line do |line|
-        next if line.strip.empty?
+    # @return [Hash] Restic's own `"summary"` fields (`"total_files"`, `"files_restored"`, ...)
+    def summary
+      @summary ||= output.messages("summary").first || {}
+    end
+    alias_method :report, :summary
 
-        hash = JSON.parse(line)
-        return hash if hash["message_type"] == "summary"
+    # @return [void]
+    def pretty_print(q)
+      q.object_group(self) do
+        q.breakable
+        q.pp summary
       end
-
-      nil
     end
   end
 
   # One `"status"` message from a live restore run.
   # https://restic.readthedocs.io/en/stable/075_scripting.html#restore
-  class RestoreStatus < SimpleDelegator
+  class RestoreStatus < Yobi::FancyHash
     # @return [Float, nil]
     def percent_done
       self["percent_done"]
@@ -140,7 +142,7 @@ module Yobi
 
   # One `"verbose_status"` message from a restore run, one per file. Only
   # emitted when `verbose: true` is passed to {Yobi::Repository#restore}.
-  class RestoreVerboseStatus < SimpleDelegator
+  class RestoreVerboseStatus < Yobi::FancyHash
     # @return [String]
     def action
       self["action"]

data/lib/yobi/repository/snapshots.rb

@@ -12,7 +12,7 @@ module Yobi
     # @param compact [Boolean] compact the printed listing
     # @param group_by [String, nil] group results, e.g. `"host"`
     # @param latest [Integer, nil] limit to the N most recent per group
-    # @return [Enumerable<Yobi::Snapshot>]
+    # @return [Array<Yobi::Snapshot>]
     def snapshots(tags: [], hosts: [], paths: [], compact: false, group_by: nil, latest: nil)
       argv = build_argv("snapshots") do |a|
         a.repeat_flag(:tag, tags)
@@ -23,34 +23,7 @@ module Yobi
         a.flag(:latest, latest) unless latest.nil?
       end
       execution = run_restic(argv)
-      Snapshots.new(execution[:output])
-    end
-  end
-
-  # Enumerable over every {Yobi::Snapshot} in one {Yobi::Repository#snapshots} call.
-  class Snapshots
-    include Enumerable
-
-    # @return [Yobi::ResticOutput]
-    attr_reader :output
-
-    # @param output [Yobi::ResticOutput]
-    def initialize(output)
-      @output = output
-    end
-
-    # @yieldparam snapshot [Yobi::Snapshot]
-    # @return [Enumerator] if no block is given
-    def each(&block)
-      return enum_for(:each) unless block_given?
-
-      entries.each(&block)
-    end
-
-    private
-
-    def entries
-      @entries ||= JSON.parse(output.to_s).map { |raw| Yobi::Snapshot.new(raw) }
+      JSON.parse(execution[:output].to_s).map { |raw| Snapshot.new(raw) }
     end
   end
 end

data/lib/yobi/repository/stats.rb

@@ -4,8 +4,7 @@ require "json"
 
 module Yobi
   class Repository
-    # Maps every valid `mode:` value for {#stats}, as both String and Symbol,
-    # to the String Restic itself expects.
+    # @private
     STATS_MODES = %w[restore-size files-by-contents blobs-per-file raw-data].each_with_object({}) do |value, hash|
       hash[value] = value
       hash[value.tr("-", "_").to_sym] = value
@@ -18,7 +17,7 @@ module Yobi
     # @param mode [String, Symbol, nil] `"restore-size"`/`:restore_size` (default), `"files-by-contents"`/`:files_by_contents`, `"blobs-per-file"`/`:blobs_per_file`, or `"raw-data"`/`:raw_data`
     # @param paths [Array<String>, String] filter by originally backed-up path(s)
     # @param tags [Array<String>, String] filter by tag(s)
-    # @return [Hash] `"total_size"`, `"total_file_count"`, `"snapshots_count"`, ...
+    # @return [Yobi::RepositoryStats]
     # @raise [ArgumentError] if `mode:` isn't one of the values listed above
     def stats(snapshot_ids: [], hosts: [], mode: nil, paths: [], tags: [])
       argv = build_argv("stats", snapshot_ids) do |a|
@@ -30,7 +29,51 @@ module Yobi
         a.repeat_flag(:tag, tags)
       end
       execution = run_restic(argv)
-      JSON.parse(execution[:output].to_s)
+      RepositoryStats.new(JSON.parse(execution[:output].to_s))
+    end
+  end
+
+  # The result of one {Yobi::Repository#stats} call.
+  # https://restic.readthedocs.io/en/stable/075_scripting.html#stats
+  class RepositoryStats < Yobi::FancyHash
+    # @return [Integer] repository size in bytes
+    def total_size
+      self["total_size"] || 0
+    end
+
+    # @return [Integer] number of files backed up in the repository
+    def total_file_count
+      self["total_file_count"] || 0
+    end
+
+    # @return [Integer] number of blobs in the repository
+    def total_blob_count
+      self["total_blob_count"] || 0
+    end
+
+    # @return [Integer] number of processed snapshots
+    def snapshots_count
+      self["snapshots_count"] || 0
+    end
+
+    # @return [Integer] repository size in bytes if blobs were uncompressed
+    def total_uncompressed_size
+      self["total_uncompressed_size"] || 0
+    end
+
+    # @return [Float] factor by which the already compressed data has shrunk due to compression
+    def compression_ratio
+      self["compression_ratio"] || 0.0
+    end
+
+    # @return [Float] percentage of already compressed data
+    def compression_progress
+      self["compression_progress"] || 0.0
+    end
+
+    # @return [Float] overall space saving due to compression
+    def compression_space_saving
+      self["compression_space_saving"] || 0.0
     end
   end
 end

data/lib/yobi/repository/tag.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "delegate"
-
 module Yobi
   class Repository
     # `restic tag`: modifies tags on existing snapshots. Tags are part of
@@ -26,56 +23,65 @@ module Yobi
         a.repeat_flag(:host, hosts)
         a.repeat_flag(:path, paths)
       end
-      execution = run_restic(argv)
-      TagOutcome.new(execution[:exit_code], execution[:output])
+      output = Yobi::ResticOutput.new(transform: Yobi::TagMessageWrapper)
+      execution = run_restic(argv, output: output)
+      TagOutcome.new(execution)
     end
   end
 
-  # The outcome of one {Yobi::Repository#tag} call.
-  class TagOutcome < Struct.new(:exit_code, :output)
-    # @return [Hash] Restic's own `"summary"` fields (`"changed_snapshots"`)
-    def report
-      @report ||= summary_hash
+  # Classifies a raw tag message Hash by its own message_type, wrapping it
+  # in the matching typed class. Used both as the {Yobi::ResticOutput}
+  # `transform:` for {Yobi::Repository#tag} and by {Yobi::TagOutcome}'s own
+  # post-hoc accessors.
+  #
+  # @private
+  module TagMessageWrapper
+    # @param raw [Hash]
+    # @return [Yobi::TagChange, Yobi::TagSummary, Hash]
+    #   the raw Hash itself for a message_type this version of Yobi doesn't recognize
+    def self.call(raw)
+      case raw["message_type"]
+      when "changed" then TagChange.new(raw)
+      when "summary" then TagSummary.new(raw)
+      else raw
+      end
     end
+  end
 
-    # @return [Array<Yobi::TagChange>] one per snapshot actually modified
-    def changes
-      @changes ||= output.each_line.filter_map do |line|
-        next if line.strip.empty?
+  # The outcome of one {Yobi::Repository#tag} call.
+  class TagOutcome
+    # @return [Yobi::ResticOutput]
+    attr_reader :output
 
-        hash = JSON.parse(line)
-        Yobi::TagChange.new(hash) if hash["message_type"] == "changed"
-      end
+    # @private
+    def initialize(execution)
+      @output = execution[:output]
     end
 
-    private
-
-    def summary_hash
-      line = output.last_line
-      hash = line && JSON.parse(line)
-      if hash && hash["message_type"] == "summary"
-        hash
-      else
-        find_summary || {}
-      end
+    # @return [Yobi::TagSummary] Restic's own `"summary"` fields (`"changed_snapshots"`)
+    def summary
+      @summary ||= output.messages("summary").first || TagSummary.new({})
     end
+    alias_method :report, :summary
 
-    def find_summary
-      output.each_line do |line|
-        next if line.strip.empty?
+    # @return [Enumerable<Yobi::TagChange>] one per snapshot actually modified
+    def changes
+      @changes ||= output.messages("changed")
+    end
 
-        hash = JSON.parse(line)
-        return hash if hash["message_type"] == "summary"
+    # @return [void]
+    def pretty_print(q)
+      q.object_group(self) do
+        q.breakable
+        q.pp summary
       end
-
-      nil
     end
   end
 
   # One `"changed"` message from a tag run. `old_snapshot_id` is now stale:
   # tags are part of a snapshot's content-addressed identity, so changing
   # them produces a new snapshot ID.
-  class TagChange < SimpleDelegator
+  class TagChange < Yobi::FancyHash
     # @return [String]
     def old_snapshot_id
       self["old_snapshot_id"]
@@ -86,4 +92,13 @@ module Yobi
       self["new_snapshot_id"]
     end
   end
+
+  # The `"summary"` message from a tag run, the final result once the
+  # command finishes. Dispatched to {Yobi::TagOutcome#summary}.
+  class TagSummary < Yobi::FancyHash
+    # @return [Integer]
+    def changed_snapshots
+      self["changed_snapshots"] || 0
+    end
+  end
 end

data/lib/yobi/repository.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "uri"
-require "json"
 
 module Yobi
   # One Restic repository. Every Restic subcommand that operates on a
@@ -9,13 +8,13 @@ module Yobi
   class Repository
     # @return [String] the repository location
     attr_reader :url
-    # @return [String, Array, Symbol, #call, nil] the repository's encryption password, as given to {#initialize}
+    # @return [String, Array, Symbol, #call] the repository's encryption password, as given to {#initialize}
     attr_reader :password
     # @return [Hash, #call] the storage backend's own credentials, as given to {#initialize}
     attr_reader :backend_credentials
 
     # @param url [String] the repository location, e.g. `"s3:s3.amazonaws.com/bucket"`
-    # @param password [String, Array, Symbol, #call, nil] a literal password; a
+    # @param password [String, Array, Symbol, #call] a literal password; a
     #   `[:command, "..."]`/`[:file, "..."]` tuple, resolved natively by Restic
     #   itself; `:insecure_no_password`; or anything responding to `#call`
     #   (invoked fresh immediately before every Restic invocation)
@@ -24,7 +23,10 @@ module Yobi
     #   responding to `#call` returning such a Hash
     # @param restic [Yobi::Restic, String, nil] a `Restic` instance to share,
     #   a bare Restic binary path, or `nil` to create a default one
-    def initialize(url:, password: nil, backend_credentials: {}, restic: nil)
+    # @raise [ArgumentError] if `password:` is `nil`
+    def initialize(url:, password:, backend_credentials: {}, restic: nil)
+      raise ArgumentError, "password: is required (use :insecure_no_password if the repository truly has none)" if password.nil?
+
       @url, @extracted_rest_credentials = extract_rest_credentials(url)
       @password = password
       @backend_credentials = backend_credentials
@@ -43,47 +45,10 @@ module Yobi
       "#<#{self.class} url=#{url.inspect} password=#{redacted_password.inspect} backend_credentials=#{redacted_backend_credentials.inspect}>"
     end
 
-    # `restic init`: creates the repository at {#url}.
-    #
-    # @param copy_chunker_params [Boolean] copy chunker parameters from `from_repo:`/`from_repository_file:`
-    # @param from_insecure_no_password [Boolean]
-    # @param from_key_hint [String, nil]
-    # @param from_password_command [String, nil]
-    # @param from_password_file [String, nil]
-    # @param from_repo [String, nil]
-    # @param from_repository_file [String, nil]
-    # @param repository_version [String, nil]
-    # @return [Hash] Restic's own `"initialized"` message
-    def init(copy_chunker_params: false, from_insecure_no_password: false, from_key_hint: nil,
-      from_password_command: nil, from_password_file: nil, from_repo: nil, from_repository_file: nil,
-      repository_version: nil)
-      argv = build_argv("init") do |a|
-        a.flag(:copy_chunker_params) if copy_chunker_params
-        a.flag(:from_insecure_no_password) if from_insecure_no_password
-        a.flag(:from_key_hint, from_key_hint) unless from_key_hint.nil?
-        a.flag(:from_password_command, from_password_command) unless from_password_command.nil?
-        a.flag(:from_password_file, from_password_file) unless from_password_file.nil?
-        a.flag(:from_repo, from_repo) unless from_repo.nil?
-        a.flag(:from_repository_file, from_repository_file) unless from_repository_file.nil?
-        a.flag(:repository_version, repository_version) unless repository_version.nil?
-      end
-      execution = run_restic(argv)
-      JSON.parse(execution[:output].to_s)
-    end
-
-    # `restic cat config`: the repository's own config document.
-    #
-    # @return [Hash]
-    # @raise [Yobi::RepositoryNotFound, Yobi::AuthenticationFailed]
-    def config
-      execution = run_restic(build_argv("cat", "config"))
-      JSON.parse(execution[:output].to_s)
-    end
-
     private
 
-    def run_restic(argv, extra_env: {}, &block)
-      @restic.run(argv, extra_env: env.merge(extra_env), &block)
+    def run_restic(argv, extra_env: {}, output: nil, &block)
+      @restic.run(argv, extra_env: env.merge(extra_env), output: output, &block)
     end
 
     def build_argv(*base)