yobi 0.1.0 → 0.3.0

data/lib/yobi/repository.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "uri"
-require "json"
 
 module Yobi
   # One Restic repository. Every Restic subcommand that operates on a
@@ -9,13 +8,13 @@ module Yobi
   class Repository
     # @return [String] the repository location
     attr_reader :url
-    # @return [String, Array, Symbol, #call, nil] the repository's encryption password, as given to {#initialize}
+    # @return [String, Array, Symbol, #call] the repository's encryption password, as given to {#initialize}
     attr_reader :password
     # @return [Hash, #call] the storage backend's own credentials, as given to {#initialize}
     attr_reader :backend_credentials
 
     # @param url [String] the repository location, e.g. `"s3:s3.amazonaws.com/bucket"`
-    # @param password [String, Array, Symbol, #call, nil] a literal password; a
+    # @param password [String, Array, Symbol, #call] a literal password; a
     #   `[:command, "..."]`/`[:file, "..."]` tuple, resolved natively by Restic
     #   itself; `:insecure_no_password`; or anything responding to `#call`
     #   (invoked fresh immediately before every Restic invocation)
@@ -24,7 +23,11 @@ module Yobi
     #   responding to `#call` returning such a Hash
     # @param restic [Yobi::Restic, String, nil] a `Restic` instance to share,
     #   a bare Restic binary path, or `nil` to create a default one
-    def initialize(url:, password: nil, backend_credentials: {}, restic: nil)
+    # @raise [ArgumentError] if `password:` is `nil`, or if `password:`/`backend_credentials:`/`restic:` is an invalid shape
+    def initialize(url:, password:, backend_credentials: {}, restic: nil)
+      validate_password_shape!(password)
+      validate_backend_credentials_shape!(backend_credentials)
+
       @url, @extracted_rest_credentials = extract_rest_credentials(url)
       @password = password
       @backend_credentials = backend_credentials
@@ -43,47 +46,10 @@ module Yobi
       "#<#{self.class} url=#{url.inspect} password=#{redacted_password.inspect} backend_credentials=#{redacted_backend_credentials.inspect}>"
     end
 
-    # `restic init`: creates the repository at {#url}.
-    #
-    # @param copy_chunker_params [Boolean] copy chunker parameters from `from_repo:`/`from_repository_file:`
-    # @param from_insecure_no_password [Boolean]
-    # @param from_key_hint [String, nil]
-    # @param from_password_command [String, nil]
-    # @param from_password_file [String, nil]
-    # @param from_repo [String, nil]
-    # @param from_repository_file [String, nil]
-    # @param repository_version [String, nil]
-    # @return [Hash] Restic's own `"initialized"` message
-    def init(copy_chunker_params: false, from_insecure_no_password: false, from_key_hint: nil,
-      from_password_command: nil, from_password_file: nil, from_repo: nil, from_repository_file: nil,
-      repository_version: nil)
-      argv = build_argv("init") do |a|
-        a.flag(:copy_chunker_params) if copy_chunker_params
-        a.flag(:from_insecure_no_password) if from_insecure_no_password
-        a.flag(:from_key_hint, from_key_hint) unless from_key_hint.nil?
-        a.flag(:from_password_command, from_password_command) unless from_password_command.nil?
-        a.flag(:from_password_file, from_password_file) unless from_password_file.nil?
-        a.flag(:from_repo, from_repo) unless from_repo.nil?
-        a.flag(:from_repository_file, from_repository_file) unless from_repository_file.nil?
-        a.flag(:repository_version, repository_version) unless repository_version.nil?
-      end
-      execution = run_restic(argv)
-      JSON.parse(execution[:output].to_s)
-    end
-
-    # `restic cat config`: the repository's own config document.
-    #
-    # @return [Hash]
-    # @raise [Yobi::RepositoryNotFound, Yobi::AuthenticationFailed]
-    def config
-      execution = run_restic(build_argv("cat", "config"))
-      JSON.parse(execution[:output].to_s)
-    end
-
     private
 
-    def run_restic(argv, extra_env: {}, &block)
-      @restic.run(argv, extra_env: env.merge(extra_env), &block)
+    def run_restic(argv, extra_env: {}, output: nil, &block)
+      @restic.run(argv, extra_env: env.merge(extra_env), output: output, &block)
     end
 
     def build_argv(*base)
@@ -154,6 +120,51 @@ module Yobi
       end
     end
 
+    def from_repo_password(from_repo)
+      from_repo.password if from_repo.is_a?(Repository)
+    end
+
+    def resolved_from_repo_flag(from_repo)
+      case from_repo
+      in Repository
+        [:from_repo, from_repo.url]
+      in String
+        [:from_repo, from_repo]
+      in [:file, String => path]
+        [:from_repository_file, path]
+      else
+        raise ArgumentError, "invalid from_repo shape"
+      end
+    end
+
+    def validate_password_shape!(value, name: "password")
+      case value
+      in nil
+        raise ArgumentError, "#{name}: is required (use :insecure_no_password if there truly is none)"
+      in :insecure_no_password | String | [:command, String] | [:file, String]
+        true
+      in callable if callable.respond_to?(:call)
+        true
+      else
+        raise ArgumentError, "invalid #{name} shape"
+      end
+    end
+
+    def validate_from_password_shape!(value)
+      validate_password_shape!(value, name: "from_password")
+    end
+
+    def validate_backend_credentials_shape!(value)
+      case value
+      in Hash
+        true
+      in callable if callable.respond_to?(:call)
+        true
+      else
+        raise ArgumentError, "invalid backend_credentials shape"
+      end
+    end
+
     def extract_rest_credentials(url)
       return [url, {}] unless url.start_with?("rest:")
 
@@ -174,6 +185,8 @@ module Yobi
         restic_or_restic_path
       in String | nil
         Yobi::Restic.new(restic_or_restic_path)
+      else
+        raise ArgumentError, "invalid restic shape"
       end
     end
   end

data/lib/yobi/restic.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "tempfile"
 require "open3"
 require "json"
 
@@ -164,10 +163,10 @@ module Yobi
 
     # `restic version`: the installed binary's own version info.
     #
-    # @return [Hash] `"version"`, `"go_version"`, `"go_os"`, `"go_arch"`
+    # @return [Yobi::ResticVersion]
     def version
       execution = run(build_argv("version"), skip_version_check: true)
-      JSON.parse(execution[:output].to_s)
+      Yobi::ResticVersion.new(parse_version_output(execution[:output].to_s))
     end
 
     # `restic cache`: lists and optionally cleans local cache directories.
@@ -193,13 +192,16 @@ module Yobi
     # @param argv [Array<String>]
     # @param extra_env [Hash{String => String}]
     # @param skip_version_check [Boolean] used internally by {#version} to avoid recursing into {#ensure_minimum_version!}
-    # @yieldparam message [Hash] a parsed JSON line, as the command runs
+    # @param output [Yobi::ResticOutput, nil] a caller-configured one (e.g. with its own `transform:`),
+    #   used instead of a plain one this creates itself; forces streaming even without a block
+    # @yieldparam message [Object] each message, live, as the command runs - transformed via
+    #   `output`'s own `transform:` if it has one, the raw parsed Hash otherwise
     # @return [Hash] `{exit_code:, output:, argv:}` on exit code 0/3
     # @raise [Yobi::RepositoryNotFound, Yobi::RepositoryLocked, Yobi::AuthenticationFailed, Yobi::ResticCommandFailed]
-    def run(argv, extra_env: {}, skip_version_check: false, &block)
+    def run(argv, extra_env: {}, skip_version_check: false, output: nil, &block)
       ensure_minimum_version! unless skip_version_check
-      execution = if block
-        execute_with_streaming(argv, extra_env, &block)
+      execution = if output || block
+        execute_with_streaming(argv, extra_env, output: output, &block)
       else
         execute(argv, extra_env)
       end
@@ -241,15 +243,14 @@ module Yobi
     # @return [Object] the block's own return value, otherwise
     def run_dump(argv, extra_env: {})
       ensure_minimum_version!
-      file = Tempfile.new("yobi-restic-output")
-      file.unlink
+      output = Yobi::ResticOutput.new
       read_end, write_end = IO.pipe
       read_end.binmode
 
-      pid = Process.spawn(env.merge(extra_env), restic_path, *argv, out: write_end, err: file)
+      pid = Process.spawn(env.merge(extra_env), restic_path, *argv, out: write_end, err: output.file)
       write_end.close
 
-      return Yobi::IOHandle.new(read_end, pid: pid, output_file: file, argv: argv) unless block_given?
+      return Yobi::IOHandle.new(read_end, pid: pid, output: output, argv: argv) unless block_given?
 
       begin
         result = yield read_end
@@ -258,10 +259,10 @@ module Yobi
         _, status = Process.wait2(pid)
       end
 
-      self.class.dispatch(exit_code: status.exitstatus, output: Yobi::ResticOutput.new(file), argv: argv)
+      self.class.dispatch(exit_code: status.exitstatus, output: output, argv: argv)
       result
     rescue Errno::ENOENT
-      file&.close
+      output.file.close
       raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
     end
 
@@ -284,6 +285,20 @@ module Yobi
 
     private
 
+    # An old enough Restic ignores --json for `version` entirely and
+    # prints plain text instead (e.g. "restic 0.9.6 compiled with
+    # go1.13.4 on linux/amd64"), which JSON.parse can't handle.
+    VERSION_LINE_PATTERN = /restic\s+(\d+\.\d+\.\d+\S*)/
+
+    def parse_version_output(raw)
+      JSON.parse(raw)
+    rescue JSON::ParserError
+      match = VERSION_LINE_PATTERN.match(raw)
+      raise unless match
+
+      {"version" => match[1]}
+    end
+
     def build_argv(*base)
       builder = ArgvBuilder.new
       builder.append(*base)
@@ -304,38 +319,64 @@ module Yobi
     end
 
     def execute(argv, extra_env)
-      file = Tempfile.new("yobi-restic-output")
-      file.unlink
-      pid = Process.spawn(env.merge(extra_env), restic_path, *argv, out: file, err: file)
+      output = Yobi::ResticOutput.new
+      pid = Process.spawn(env.merge(extra_env), restic_path, *argv, out: output.file, err: output.file)
       _, status = Process.wait2(pid)
-      {exit_code: status.exitstatus, output: Yobi::ResticOutput.new(file), argv: argv}
+      {exit_code: status.exitstatus, output: output, argv: argv}
     rescue Errno::ENOENT
-      file&.close
+      output.file.close
       raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
     end
 
-    def execute_with_streaming(argv, extra_env)
-      file = Tempfile.new("yobi-restic-output")
-      file.unlink
+    def execute_with_streaming(argv, extra_env, output: nil)
+      output ||= Yobi::ResticOutput.new
 
-      Open3.popen2e(env.merge(extra_env), restic_path, *argv) do |stdin, merged_output, wait_thr|
+      Open3.popen3(env.merge(extra_env), restic_path, *argv) do |stdin, stdout, stderr, wait_thr|
         stdin.close
-        merged_output.each_line do |line|
-          file.write(line)
-          parsed = parse_streamed_line(line)
-          yield parsed if parsed
+        readers = {stdout => :stdout, stderr => :stderr}
+
+        until readers.empty?
+          ready, = IO.select(readers.keys)
+          ready.each do |io|
+            line = io.gets
+            if line.nil?
+              readers.delete(io)
+              next
+            end
+
+            parsed = output.write_and_parse(line, readers[io])
+            yield parsed if parsed && block_given?
+          end
         end
-        {exit_code: wait_thr.value.exitstatus, output: Yobi::ResticOutput.new(file), argv: argv}
+
+        {exit_code: wait_thr.value.exitstatus, output: output, argv: argv}
       end
     rescue Errno::ENOENT
-      file&.close
+      output.file.close
       raise Yobi::ResticNotFound.new(restic_path: restic_path, argv: argv)
     end
+  end
+
+  # The result of one {Yobi::Restic#version} call.
+  class ResticVersion < Yobi::FancyHash
+    # @return [String]
+    def version
+      self["version"]
+    end
 
-    def parse_streamed_line(line)
-      JSON.parse(line)
-    rescue JSON::ParserError
-      nil
+    # @return [String, nil]
+    def go_version
+      self["go_version"]
+    end
+
+    # @return [String, nil]
+    def go_os
+      self["go_os"]
+    end
+
+    # @return [String, nil]
+    def go_arch
+      self["go_arch"]
     end
   end
 end

data/lib/yobi/restic_output.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-module Yobi
-  # Captures a line's message_type without fully parsing it as JSON.
-  #
-  # @private
-  MESSAGE_TYPE_LINE_PATTERN = /"message_type"\s*:\s*"(\w+)"/
+require "json"
+require "tempfile"
 
+module Yobi
   # Wraps the tempfile a repository operation's stdout+stderr are both
   # written to, with random access so a caller asking only for the last
   # line (see {#last_line}) doesn't have to read the entire output.
@@ -15,11 +13,19 @@ module Yobi
     # Default read window size, in bytes, for {#last_line}.
     DEFAULT_TAIL_CHUNK_SIZE = 4096
 
-    # @param file [File]
+    # @return [File] a fresh, already-unlinked tempfile of its own
+    attr_reader :file
+
     # @param tail_chunk_size [Integer] read window size for {#last_line}
-    def initialize(file, tail_chunk_size: DEFAULT_TAIL_CHUNK_SIZE)
-      @file = file
+    # @param transform [Proc, nil] used by {#write_and_parse} during a streaming run to transform
+    #   each message, and as {#messages}' default transform afterward, when no explicit one is given
+    def initialize(tail_chunk_size: DEFAULT_TAIL_CHUNK_SIZE, transform: nil)
+      @file = Tempfile.new("yobi-restic-output")
+      @file.unlink
       @tail_chunk_size = tail_chunk_size
+      @transform = transform
+      @index = nil
+      @stderr_offsets = []
     end
 
     # Byte offsets of every line, grouped by message_type. Built once and
@@ -69,26 +75,177 @@ module Yobi
       @file.each_line(&block)
     end
 
+    # Every message recorded in {#index}, restricted to one +message_type+
+    # if given, in file order. Without an explicit transform block, uses
+    # the +transform:+ given to {#initialize} (if any); with neither, just
+    # yields/returns the raw parsed Hash. Whenever a transform (explicit
+    # or the stored one) applies, wraps the result in a {LazyList}
+    # instead of yielding.
+    #
+    # @param message_type [String, nil] every message, if omitted
+    # @yieldparam message [Hash] the raw parsed message, to transform
+    # @return [Yobi::ResticOutput::LazyList] if a transform block (or a stored transform) applies
+    # @return [Enumerator] of raw Hashes, otherwise
+    def messages(message_type = nil, &transform)
+      transform ||= @transform
+      return raw_messages(message_type) unless transform
+
+      offsets = message_type ? index[message_type] : index.values.flatten.sort
+      LazyList.new(self, offsets, transform)
+    end
+
+    # Every line known to have arrived on stderr rather than stdout during
+    # a streaming run (see {#write_and_parse}), in file order.
+    #
+    # @yieldparam line [String]
+    # @return [Enumerator] if no block is given
+    def stderr_lines
+      return enum_for(:stderr_lines) unless block_given?
+
+      @stderr_offsets.each { |offset| yield read_line_at(offset) }
+    end
+
+    # Called once per line read from a live streaming run's stdout/stderr.
+    # Writes it to the file, parses it, and - if it's a JSON message -
+    # indexes it by message_type and returns it transformed via the
+    # +transform:+ given to {#initialize} (or the raw parsed Hash, if none
+    # was given). A non-JSON line's offset is recorded in {#stderr_lines}
+    # instead, if +origin+ is +:stderr+; either way, returns nil.
+    #
+    # @param line [String]
+    # @param origin [:stdout, :stderr]
+    # @return [Object, nil]
+    def write_and_parse(line, origin)
+      offset = @file.pos
+      @file.write(line)
+
+      parsed = begin
+        JSON.parse(line)
+      rescue JSON::ParserError
+        nil
+      end
+
+      if parsed
+        (@index ||= Hash.new { |hash, message_type| hash[message_type] = [] })[parsed["message_type"]] << offset if parsed["message_type"]
+        @transform ? @transform.call(parsed) : parsed
+      elsif origin == :stderr
+        @stderr_offsets << offset
+        nil
+      end
+    end
+
     # @return [String] the full captured output
     def to_s
       @file.rewind
       @file.read
     end
 
+    # @return [String]
+    def inspect
+      "#<#{self.class} #{@file.size} bytes>"
+    end
+
     private
 
-    def build_index
-      index = Hash.new { |hash, message_type| hash[message_type] = [] }
+    def raw_messages(message_type)
+      return enum_for(:raw_messages, message_type) unless block_given?
+
+      if message_type
+        index[message_type].each { |offset| yield JSON.parse(read_line_at(offset)) }
+      else
+        each_parsed_line { |_offset, parsed| yield parsed }
+      end
+    end
+
+    def each_parsed_line
       @file.rewind
       loop do
         offset = @file.pos
         line = @file.gets
         break unless line
 
-        match = MESSAGE_TYPE_LINE_PATTERN.match(line)
-        index[match[1]] << offset if match
+        parsed = begin
+          JSON.parse(line)
+        rescue JSON::ParserError
+          nil
+        end
+        yield(offset, parsed) if parsed && parsed["message_type"]
       end
+    end
+
+    def build_index
+      index = Hash.new { |hash, message_type| hash[message_type] = [] }
+      each_parsed_line { |offset, parsed| index[parsed["message_type"]] << offset }
       index
     end
+
+    # A lazily-transformed view over a set of {#index} offsets, printing
+    # like a plain Array (bounded to a preview, unlike one). Returned by
+    # {ResticOutput#messages} when given a transform block.
+    # #inspect/#pretty_print's preview is memoized, and #each reuses it
+    # rather than re-reading/re-transforming those same offsets again
+    # when the caller iterates the whole thing.
+    #
+    # @private
+    class LazyList
+      include Enumerable
+
+      # Items #inspect/#pretty_print show before truncating with "...".
+      # One louder than ActiveRecord::Relation#inspect's own preview size.
+      INSPECT_PREVIEW_SIZE = 11
+
+      # @param output [Yobi::ResticOutput]
+      # @param offsets [Array<Integer>]
+      # @param transform [Proc] applied to each offset's raw parsed Hash
+      def initialize(output, offsets, transform)
+        @output = output
+        @offsets = offsets
+        @transform = transform
+      end
+
+      # @yieldparam item [Object]
+      # @return [Enumerator] if no block is given
+      def each
+        return enum_for(:each) unless block_given?
+
+        preview.each { |item| yield item }
+        @offsets[preview.size..].each { |offset| yield transform_at(offset) }
+      end
+
+      # The number of messages, known upfront from {#index} - unlike
+      # +Enumerable#count+, doesn't read/transform a single one to answer.
+      #
+      # @return [Integer]
+      def size
+        @offsets.size
+      end
+      alias_method :length, :size
+
+      # @return [String]
+      def inspect
+        shown = preview.first(INSPECT_PREVIEW_SIZE).map(&:inspect).join(", ")
+        shown += ", ..." if preview.size > INSPECT_PREVIEW_SIZE
+        "[#{shown}]"
+      end
+
+      # @return [void]
+      def pretty_print(q)
+        q.group(1, "[", "]") do
+          q.seplist(preview.first(INSPECT_PREVIEW_SIZE)) { |item| q.pp item }
+          q.text ", ..." if preview.size > INSPECT_PREVIEW_SIZE
+        end
+      end
+
+      private
+
+      # @return [Array<Object>]
+      def preview
+        @preview ||= @offsets.first(INSPECT_PREVIEW_SIZE + 1).map { |offset| transform_at(offset) }
+      end
+
+      def transform_at(offset)
+        @transform.call(JSON.parse(@output.read_line_at(offset)))
+      end
+    end
   end
 end

data/lib/yobi/snapshot.rb

@@ -1,12 +1,11 @@
 # frozen_string_literal: true
 
 require "time"
-require "delegate"
 
 module Yobi
   # One snapshot, as Restic reports it. Shared across {Yobi::Repository#snapshots},
   # {Yobi::Repository#ls}, and {Yobi::Repository#forget}'s keep/remove entries.
-  class Snapshot < SimpleDelegator
+  class Snapshot < Yobi::FancyHash
     # @return [String]
     def id
       self["id"]
@@ -41,5 +40,85 @@ module Yobi
     def parent_id
       self["parent"]
     end
+
+    # @return [Yobi::SnapshotSummary]
+    def summary
+      @summary ||= SnapshotSummary.new(self["summary"] || {})
+    end
+  end
+
+  # The `"summary"` field of a {Yobi::Snapshot}, the stats recorded when it
+  # was created.
+  # https://restic.readthedocs.io/en/stable/075_scripting.html#snapshotsummary-object
+  class SnapshotSummary < Yobi::FancyHash
+    # @return [Time, nil]
+    def backup_start
+      @backup_start ||= Time.parse(self["backup_start"]) if self["backup_start"]
+    end
+
+    # @return [Time, nil]
+    def backup_end
+      @backup_end ||= Time.parse(self["backup_end"]) if self["backup_end"]
+    end
+
+    # @return [Integer]
+    def files_new
+      self["files_new"] || 0
+    end
+
+    # @return [Integer]
+    def files_changed
+      self["files_changed"] || 0
+    end
+
+    # @return [Integer]
+    def files_unmodified
+      self["files_unmodified"] || 0
+    end
+
+    # @return [Integer]
+    def dirs_new
+      self["dirs_new"] || 0
+    end
+
+    # @return [Integer]
+    def dirs_changed
+      self["dirs_changed"] || 0
+    end
+
+    # @return [Integer]
+    def dirs_unmodified
+      self["dirs_unmodified"] || 0
+    end
+
+    # @return [Integer]
+    def data_blobs
+      self["data_blobs"] || 0
+    end
+
+    # @return [Integer]
+    def tree_blobs
+      self["tree_blobs"] || 0
+    end
+
+    # @return [Integer]
+    def data_added
+      self["data_added"] || 0
+    end
+
+    # @return [Integer]
+    def data_added_packed
+      self["data_added_packed"] || 0
+    end
+
+    # @return [Integer]
+    def total_files_processed
+      self["total_files_processed"] || 0
+    end
+
+    # @return [Integer]
+    def total_bytes_processed
+      self["total_bytes_processed"] || 0
+    end
   end
 end

data/lib/yobi/version.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-# A Ruby library for the Restic backup tool, wrapping the `restic` CLI in
+# A Ruby library for the Restic backup program, wrapping the `restic` CLI in
 # plain Ruby objects instead of shelling out to flags and raw JSON by hand.
 module Yobi
   # @return [String]
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/yobi.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 require_relative "yobi/version"
+require_relative "yobi/fancy_hash"
 require_relative "yobi/errors"
 require_relative "yobi/argv_builder"
 require_relative "yobi/restic_output"
 require_relative "yobi/io_handle"
-require_relative "yobi/mount"
+require_relative "yobi/mount_handle"
 require_relative "yobi/restic"
 require_relative "yobi/snapshot"
 require_relative "yobi/repository"