kamal-backup 0.3.0.beta5 → 0.3.0.beta7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b92197e0f2d480ca2cebd5e2505304686b739ca0ed4c61c24b4b4caca39054d
-  data.tar.gz: ff2b00e495d6690956d1ed8070bc43989ca1a9e4b9846b6503ababa26adbd01f
+  metadata.gz: 841b29d2ee8cc4598cb1880da4a7bf884cef141816432a9d960ff400b7535d25
+  data.tar.gz: 3784f1bfe50f1480f457bfb40a18e30ab5905db4a300a193fe5d68ac50d67f2f
 SHA512:
-  metadata.gz: c8f93d59add03f4bf66b48ae0ea61cf46c7682c527ac604fcbd98a8c0790e6cc77d9d45759fabbfb5d6fcbce25da0e82b421f56cc8771820c57b1933539634c2
-  data.tar.gz: ebd54357f4bdc1829a71cb76fcef4e60000661ce7eb8f4d1a974fb22438b1054fe96bfa26095509d74fd50e5a13d5e234fa0c845440c8f33ddfd49e798b307f4
+  metadata.gz: 9929349441963b8297254c10faa1446cf4f7602638a7637ab655ffcadb32d31d012ea8827890fc29175dfc4e344a5afc1edacbf960cfe86d6aec2a83c88385c6
+  data.tar.gz: fbfee00eae64424a92e5d039512ff51d01abecb6dd6a61d8d44ec97dc2e86208f06fd5c191947107a9c9adaa4da80a77a9eebabc9e9e12b96d7e60dcb0ce0e06

data/lib/kamal_backup/cli.rb

@@ -20,11 +20,17 @@ module KamalBackup
       end
 
       def direct_app
-        @direct_app ||= App.new(config: Config.new(env: command_env), redactor: redactor)
+        @direct_app ||= App.new(
+          config: Config.new(env: command_env),
+          redactor: redactor
+        )
       end
 
-      def local_app
-        @local_app ||= App.new(config: local_command_config, redactor: redactor)
+      def local_restore_app
+        @local_restore_app ||= App.new(
+          config: local_command_config,
+          redactor: redactor
+        )
       end
 
       def local_preferences
@@ -65,7 +71,9 @@ module KamalBackup
           redactor: redactor,
           config_file: options[:config_file],
           destination: options[:destination],
-          env: command_env
+          env: command_env,
+          stdout: $stdout,
+          stderr: $stderr
         )
       end
 
@@ -94,9 +102,11 @@ module KamalBackup
 
         result = bridge.execute_on_accessory(
           accessory_name: accessory_name,
-          command: Shellwords.join(argv)
+          command: Shellwords.join(argv),
+          stream: true
         )
-        print(result.stdout)
+        print(result.stdout) unless result.streamed
+        $stderr.print(result.stderr) if !result.streamed && !result.stderr.empty?
         result
       end
 
@@ -276,7 +286,7 @@ module KamalBackup
       desc "local [SNAPSHOT]", "Restore the backup into the local database and Active Storage path"
       def local(snapshot = "latest")
         confirm!("Restore #{snapshot} into the local database and Active Storage path? This will overwrite local data.")
-        puts(JSON.pretty_generate(local_app.restore_to_local_machine(snapshot)))
+        puts(JSON.pretty_generate(local_restore_app.restore_to_local_machine(snapshot)))
       end
 
       method_option :"confirm-production-restore", type: :boolean, default: false, desc: "Confirm production restore without interactive prompts"
@@ -301,9 +311,9 @@ module KamalBackup
       desc "local [SNAPSHOT]", "Run a restore drill on the local machine"
       def local(snapshot = "latest")
         confirm!("Run a local restore drill for #{snapshot}? This will overwrite local data.")
-        result = local_app.drill_on_local_machine(snapshot, check_command: options[:check])
+        result = local_restore_app.drill_on_local_machine(snapshot, check_command: options[:check])
         puts(JSON.pretty_generate(result))
-        exit(1) if local_app.drill_failed?(result)
+        exit(1) if local_restore_app.drill_failed?(result)
       end
 
       method_option :database, type: :string, desc: "Scratch database name for PostgreSQL or MySQL"
@@ -389,7 +399,9 @@ module KamalBackup
 
     def self.start(argv = ARGV, env: ENV)
       self.command_env = env
-      super(normalize_global_options(argv))
+      Command.with_output(CommandOutput.new(io: $stderr)) do
+        super(normalize_global_options(argv))
+      end
     rescue Error => e
       warn("kamal-backup: #{Redactor.new(env: env).redact_string(e.message)}")
       exit(1)

data/lib/kamal_backup/command.rb

@@ -1,4 +1,6 @@
 require "open3"
+require "securerandom"
+require "shellwords"
 require_relative "errors"
 
 module KamalBackup
@@ -18,14 +20,185 @@ module KamalBackup
 
     def display(redactor)
       env_prefix = env.keys.sort.map { |key| "#{key}=#{redactor.redact_value(key, env[key])}" }
-      redactor.redact_string((env_prefix + argv).join(" "))
+      redactor.redact_string((env_prefix + [argv.shelljoin]).join(" "))
     end
   end
 
-  CommandResult = Struct.new(:stdout, :stderr, :status, keyword_init: true)
+  CommandResult = Struct.new(:stdout, :stderr, :status, :streamed, keyword_init: true)
+
+  class CommandOutput
+    LEVELS = {
+      "DEBUG" => 0,
+      "INFO" => 1,
+      "WARN" => 2,
+      "ERROR" => 3,
+      "FATAL" => 4
+    }.freeze
+    LEVEL_COLORS = {
+      "DEBUG" => :black,
+      "INFO" => :blue,
+      "WARN" => :yellow,
+      "ERROR" => :red,
+      "FATAL" => :red
+    }.freeze
+    COLOR_CODES = {
+      black: 30,
+      red: 31,
+      green: 32,
+      yellow: 33,
+      blue: 34,
+      magenta: 35,
+      cyan: 36,
+      white: 37,
+      light_black: 90,
+      light_red: 91,
+      light_green: 92,
+      light_yellow: 93,
+      light_blue: 94,
+      light_magenta: 95,
+      light_cyan: 96,
+      light_white: 97
+    }.freeze
+
+    def initialize(io: $stdout, env: ENV, verbosity: :info)
+      @io = io
+      @env = env
+      @verbosity = LEVELS.fetch(verbosity.to_s.upcase)
+      @mutex = Mutex.new
+      @buffers = {}
+    end
+
+    def info(message, redactor:)
+      write_message("INFO", redactor.redact_string(message))
+    end
+
+    def command_start(spec, redactor:)
+      id = SecureRandom.hex(4)
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      display = spec.display(redactor)
+
+      write_message("INFO", "Running #{colorize(display, :yellow, :bold)} #{local_target}", id)
+      write_message("DEBUG", "Command: #{colorize(display, :blue)}", id)
+
+      { id: id, started_at: started_at, redactor: redactor }
+    end
+
+    def command_output(context, _stream, data, redactor:)
+      raw = data.to_s
+      return if raw.empty?
+      return unless log_level?("DEBUG")
+
+      synchronize do
+        key = [context.fetch(:id), _stream]
+        @buffers[key] = "#{@buffers[key]}#{raw}"
+        flush_complete_output_lines(context, key, redactor: redactor)
+      end
+    end
+
+    def command_exit(context, status)
+      runtime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - context.fetch(:started_at)
+      result = status.to_i.zero? ? "successful" : "failed"
+      result_color = status.to_i.zero? ? :green : :red
+
+      synchronize do
+        flush_output_buffers(context)
+        write_message_unlocked("INFO", "Finished in #{format("%.3f seconds", runtime)} with exit status #{status} (#{colorize(result, result_color, :bold)}).", context.fetch(:id))
+      end
+    end
+
+    private
+      def write_message(level, message, id = nil)
+        return unless log_level?(level)
+
+        synchronize { write_message_unlocked(level, message, id) }
+      end
+
+      def write_message_unlocked(level, message, id = nil)
+        @io.puts(format_message(level, message, id)) if log_level?(level)
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
+      def flush_complete_output_lines(context, key, redactor:)
+        buffer = @buffers.fetch(key)
+        output = +""
+
+        while (index = buffer.index("\n"))
+          output << buffer.slice!(0..index)
+        end
+
+        @buffers[key] = buffer
+        write_output(context, output, redactor: redactor, stream: key.last) unless output.empty?
+      end
+
+      def flush_output_buffers(context)
+        id = context.fetch(:id)
+        keys = @buffers.keys.select { |key_id, _stream| key_id == id }
+
+        keys.each do |key|
+          output = @buffers.delete(key)
+          next if output.to_s.empty?
+
+          write_output(context, output, redactor: context.fetch(:redactor), stream: key.last)
+        end
+      end
+
+      def write_output(context, output, redactor:, stream: nil)
+        color = stream == :stderr ? :red : :green
+
+        redactor.redact_string(output).each_line do |line|
+          write_message_unlocked("DEBUG", colorize("\t#{line}".chomp, color), context.fetch(:id))
+        end
+        @io.flush if @io.respond_to?(:flush)
+      end
+
+      def format_message(level, message, id = nil)
+        message = "[#{colorize(id, :green)}] #{message}" if id
+        "#{colorize(level.rjust(6), LEVEL_COLORS.fetch(level))} #{message}"
+      end
+
+      def local_target
+        user = @env["USER"].to_s.empty? ? @env["USERNAME"].to_s : @env["USER"].to_s
+
+        if user.empty?
+          "on #{colorize("localhost", :blue)}"
+        else
+          "as #{colorize(user, :blue)}@#{colorize("localhost", :blue)}"
+        end
+      end
+
+      def log_level?(level)
+        LEVELS.fetch(level) >= @verbosity
+      end
+
+      def colorize(value, color, mode = nil)
+        string = value.to_s
+        return string unless colorize?
+        return string unless COLOR_CODES.key?(color)
+
+        prefix = mode == :bold ? "\e[1;" : "\e[0;"
+        "#{prefix}#{COLOR_CODES.fetch(color)};49m#{string}\e[0m"
+      end
+
+      def colorize?
+        @env["SSHKIT_COLOR"] || (@io.respond_to?(:tty?) && @io.tty?)
+      end
+  end
 
   class Command
     class << self
+      attr_accessor :output
+
+      def with_output(output)
+        previous_output = self.output
+        self.output = output
+        yield
+      ensure
+        self.output = previous_output
+      end
+
       def available?(name)
         ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).any? do |dir|
           path = File.join(dir, name)
@@ -33,9 +206,23 @@ module KamalBackup
         end
       end
 
-      def capture(spec, input: nil, redactor:)
-        stdout, stderr, status = Open3.capture3(spec.env, *spec.argv, stdin_data: input)
-        result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus)
+      def capture(spec, input: nil, redactor:, log: true, log_output: true, tee_stdout: nil, tee_stderr: nil)
+        output = log ? self.output : nil
+        return capture_quietly(spec, input: input, redactor: redactor) unless output || tee_stdout || tee_stderr
+
+        context = output&.command_start(spec, redactor: redactor)
+        stdout, stderr, status = popen_capture(
+          spec,
+          input: input,
+          redactor: redactor,
+          output: output,
+          context: context,
+          log_output: log_output,
+          tee_stdout: tee_stdout,
+          tee_stderr: tee_stderr
+        )
+        output&.command_exit(context, status.exitstatus)
+        result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus, streamed: !!(tee_stdout || tee_stderr))
 
         if status.success?
           result
@@ -46,7 +233,72 @@ module KamalBackup
         raise command_not_found(spec, e)
       end
 
+      def collect_stream(io, command_output: self.output, context: nil, stream: :stdout, log_output: true, tee_io: nil, redactor: nil)
+        captured_output = +""
+        tee_buffer = +""
+
+        loop do
+          chunk = io.readpartial(16 * 1024)
+          captured_output << chunk
+          command_output&.command_output(context, stream, chunk, redactor: redactor) if log_output && context
+          tee_buffer = tee_stream(tee_io, redactor, tee_buffer, chunk) if tee_io
+        rescue EOFError
+          flush_tee_stream(tee_io, redactor, tee_buffer) if tee_io
+          break
+        end
+
+        captured_output
+      end
+
       private
+        def tee_stream(io, redactor, buffer, chunk)
+          buffer << chunk
+
+          while (index = buffer.index("\n"))
+            io.print(redactor.redact_string(buffer.slice!(0..index)))
+          end
+
+          io.flush if io.respond_to?(:flush)
+          buffer
+        end
+
+        def flush_tee_stream(io, redactor, buffer)
+          return if buffer.empty?
+
+          io.print(redactor.redact_string(buffer))
+          io.flush if io.respond_to?(:flush)
+        end
+
+        def capture_quietly(spec, input:, redactor:)
+          stdout, stderr, status = Open3.capture3(spec.env, *spec.argv, stdin_data: input)
+          result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus, streamed: false)
+
+          if status.success?
+            result
+          else
+            raise command_failure(spec, status.exitstatus, stdout, stderr, redactor)
+          end
+        end
+
+        def popen_capture(spec, input:, redactor:, output:, context:, log_output:, tee_stdout:, tee_stderr:)
+          Open3.popen3(spec.env, *spec.argv) do |stdin, stdout, stderr, wait_thread|
+            stdin_writer = Thread.new do
+              stdin.write(input) if input
+            ensure
+              stdin.close unless stdin.closed?
+            end
+            stdout_reader = Thread.new do
+              collect_stream(stdout, command_output: output, context: context, stream: :stdout, log_output: log_output, tee_io: tee_stdout, redactor: redactor)
+            end
+            stderr_reader = Thread.new do
+              collect_stream(stderr, command_output: output, context: context, stream: :stderr, log_output: log_output, tee_io: tee_stderr, redactor: redactor)
+            end
+
+            stdin_writer.join
+            [stdout_reader.value, stderr_reader.value, wait_thread.value]
+          end
+        end
+
         def command_failure(spec, status, stdout, stderr, redactor)
           CommandError.new(
             "command failed (#{status}): #{spec.display(redactor)}\n#{redactor.redact_string(stderr)}",

data/lib/kamal_backup/kamal_bridge.rb

@@ -7,12 +7,14 @@ module KamalBackup
     DEFAULT_CONFIG_FILE = "config/deploy.yml"
     VERSION_LINE_PATTERN = /\A\d+(?:\.\d+)+(?:[-.][A-Za-z0-9]+)*\z/
 
-    def initialize(redactor:, config_file: nil, destination: nil, env: ENV, cwd: Dir.pwd)
+    def initialize(redactor:, config_file: nil, destination: nil, env: ENV, cwd: Dir.pwd, stdout: $stdout, stderr: $stderr)
       @redactor = redactor
       @config_file = config_file
       @destination = destination
       @env = env
       @cwd = cwd
+      @stdout = stdout
+      @stderr = stderr
     end
 
     def accessory_name(preferred: nil)
@@ -50,8 +52,8 @@ module KamalBackup
       accessory_secret_placeholders(accessory_name).merge(accessory_clear_env(accessory_name))
     end
 
-    def execute_on_accessory(accessory_name:, command:)
-      capture_kamal(kamal_exec_argv(accessory_name, command))
+    def execute_on_accessory(accessory_name:, command:, stream: false)
+      capture_kamal(kamal_exec_argv(accessory_name, command), stream: stream)
     end
 
     def remote_version(accessory_name:)
@@ -212,13 +214,20 @@ module KamalBackup
         argv
       end
 
-      def capture_kamal(argv)
+      def capture_kamal(argv, stream: false)
         spec = CommandSpec.new(argv: argv)
+        options = {
+          redactor: @redactor,
+          log: false,
+          log_output: false,
+          tee_stdout: stream ? @stdout : nil,
+          tee_stderr: stream ? @stderr : nil
+        }
 
         if defined?(Bundler)
-          Bundler.with_unbundled_env { Command.capture(spec, redactor: @redactor) }
+          Bundler.with_unbundled_env { Command.capture(spec, **options) }
         else
-          Command.capture(spec, redactor: @redactor)
+          Command.capture(spec, **options)
         end
       end
 

data/lib/kamal_backup/restic.rb

@@ -16,7 +16,7 @@ module KamalBackup
     end
 
     def ensure_repository
-      run(%w[snapshots --json])
+      run(%w[snapshots --json], log_output: false)
     rescue CommandError => e
       if config.restic_init_if_missing?
         log("restic repository not ready, running restic init")
@@ -43,14 +43,17 @@ module KamalBackup
       log("backing up file content as #{filename}")
 
       File.open(path, "rb") do |file|
+        output = Command.output
+        context = output&.command_start(command, redactor: redactor)
         Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
-          stdout_reader = Thread.new { stdout.read }
-          stderr_reader = Thread.new { stderr.read }
+          stdout_reader = Thread.new { Command.collect_stream(stdout, command_output: output, context: context, stream: :stdout, redactor: redactor) }
+          stderr_reader = Thread.new { Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr, redactor: redactor) }
           IO.copy_stream(file, stdin)
           stdin.close
           out = stdout_reader.value
           err = stderr_reader.value
           status = wait_thread.value
+          output&.command_exit(context, status.exitstatus)
           raise_command_error(command, status, out, err) unless status.success?
 
           CommandResult.new(stdout: out, stderr: err, status: status.exitstatus)
@@ -99,7 +102,7 @@ module KamalBackup
     end
 
     def snapshots_json(tags: common_tags)
-      output = run(["snapshots", "--json"] + filter_tag_args(tags)).stdout
+      output = run(["snapshots", "--json"] + filter_tag_args(tags), log_output: false).stdout
       snapshots = JSON.parse(output)
       required_tags = tags.compact
       snapshots.select do |snapshot|
@@ -116,7 +119,7 @@ module KamalBackup
     end
 
     def ls_json(snapshot)
-      output = run(["ls", "--json", snapshot]).stdout
+      output = run(["ls", "--json", snapshot], log_output: false).stdout
       output.lines.filter_map do |line|
         JSON.parse(line)
       rescue JSON::ParserError
@@ -153,12 +156,15 @@ module KamalBackup
       FileUtils.mkdir_p(File.dirname(target_path))
       temp_path = "#{target_path}.kamal-backup-#{$$}.tmp"
 
+      output = Command.output
+      context = output&.command_start(command, redactor: redactor)
       Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
         stdin.close
-        stderr_reader = Thread.new { stderr.read }
+        stderr_reader = Thread.new { Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr, redactor: redactor) }
         File.open(temp_path, "wb") { |file| IO.copy_stream(stdout, file) }
         err = stderr_reader.value
         status = wait_thread.value
+        output&.command_exit(context, status.exitstatus)
         raise_command_error(command, status, "", err) unless status.success?
       end
       File.rename(temp_path, target_path)
@@ -176,8 +182,12 @@ module KamalBackup
       run(["restore", snapshot, "--target", target])
     end
 
-    def run(args)
-      Command.capture(CommandSpec.new(argv: ["restic"] + args, env: restic_env), redactor: redactor)
+    def run(args, log_output: true)
+      Command.capture(
+        CommandSpec.new(argv: ["restic"] + args, env: restic_env),
+        redactor: redactor,
+        log_output: log_output
+      )
     end
 
     def common_tags
@@ -240,13 +250,16 @@ module KamalBackup
       end
 
       def pipe_commands(producer, consumer, producer_label:, consumer_label:)
+        output = Command.output
+        producer_context = output&.command_start(producer, redactor: redactor)
         Open3.popen3(producer.env, *producer.argv) do |producer_stdin, producer_stdout, producer_stderr, producer_wait|
           producer_stdin.close
 
+          consumer_context = output&.command_start(consumer, redactor: redactor)
           Open3.popen3(consumer.env, *consumer.argv) do |consumer_stdin, consumer_stdout, consumer_stderr, consumer_wait|
-            producer_err_reader = Thread.new { producer_stderr.read }
-            consumer_out_reader = Thread.new { consumer_stdout.read }
-            consumer_err_reader = Thread.new { consumer_stderr.read }
+            producer_err_reader = Thread.new { Command.collect_stream(producer_stderr, command_output: output, context: producer_context, stream: :stderr, redactor: redactor) }
+            consumer_out_reader = Thread.new { Command.collect_stream(consumer_stdout, command_output: output, context: consumer_context, stream: :stdout, redactor: redactor) }
+            consumer_err_reader = Thread.new { Command.collect_stream(consumer_stderr, command_output: output, context: consumer_context, stream: :stderr, redactor: redactor) }
 
             copy_error = nil
             copy_thread = Thread.new do
@@ -260,6 +273,8 @@ module KamalBackup
             copy_thread.join
             producer_status = producer_wait.value
             consumer_status = consumer_wait.value
+            output&.command_exit(producer_context, producer_status.exitstatus)
+            output&.command_exit(consumer_context, consumer_status.exitstatus)
 
             producer_err = producer_err_reader.value
             consumer_out = consumer_out_reader.value
@@ -302,7 +317,11 @@ module KamalBackup
       end
 
       def log(message)
-        $stdout.puts("[kamal-backup] #{redactor.redact_string(message)}")
+        if Command.output
+          Command.output.info(message, redactor: redactor)
+        else
+          $stdout.puts("[kamal-backup] #{redactor.redact_string(message)}")
+        end
       end
   end
 end

data/lib/kamal_backup/version.rb

@@ -1,3 +1,3 @@
 module KamalBackup
-  VERSION = "0.3.0.beta5"
+  VERSION = "0.3.0.beta7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kamal-backup
 version: !ruby/object:Gem::Version
-  version: 0.3.0.beta5
+  version: 0.3.0.beta7
 platform: ruby
 authors:
 - crmne