kamal-backup 0.3.0.beta5 → 0.3.0.beta6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b92197e0f2d480ca2cebd5e2505304686b739ca0ed4c61c24b4b4caca39054d
-  data.tar.gz: ff2b00e495d6690956d1ed8070bc43989ca1a9e4b9846b6503ababa26adbd01f
+  metadata.gz: 68d3201dc26c940938479dc76d1c0e17b206676ae69610d736a69879a78de21c
+  data.tar.gz: dd58ee38da83d4652515b108869e5fe441a25d80697b7db3ea1c606240a44766
 SHA512:
-  metadata.gz: c8f93d59add03f4bf66b48ae0ea61cf46c7682c527ac604fcbd98a8c0790e6cc77d9d45759fabbfb5d6fcbce25da0e82b421f56cc8771820c57b1933539634c2
-  data.tar.gz: ebd54357f4bdc1829a71cb76fcef4e60000661ce7eb8f4d1a974fb22438b1054fe96bfa26095509d74fd50e5a13d5e234fa0c845440c8f33ddfd49e798b307f4
+  metadata.gz: 37ca5f6463fe6eb76c2f6994a783e2180f3eecff24e834550743c1f9130e6ad3019ce1e296dc44ac523a36c065f1e05ac00aaaec7c7224254daea0c2d683d7a0
+  data.tar.gz: '0985cd5e2eb8a52db5dcc4265271eddde1513c8c8db80d0aec21ee3ca1be0bd42e2adcd60ffeb942aa7bb2d926cbad69f5d89051485ce64145f3f5c821abe3bb'

data/lib/kamal_backup/cli.rb

@@ -20,11 +20,17 @@ module KamalBackup
       end
 
       def direct_app
-        @direct_app ||= App.new(config: Config.new(env: command_env), redactor: redactor)
+        @direct_app ||= App.new(
+          config: Config.new(env: command_env),
+          redactor: redactor
+        )
       end
 
-      def local_app
-        @local_app ||= App.new(config: local_command_config, redactor: redactor)
+      def local_restore_app
+        @local_restore_app ||= App.new(
+          config: local_command_config,
+          redactor: redactor
+        )
       end
 
       def local_preferences
@@ -65,7 +71,9 @@ module KamalBackup
           redactor: redactor,
           config_file: options[:config_file],
           destination: options[:destination],
-          env: command_env
+          env: command_env,
+          stdout: $stdout,
+          stderr: $stderr
         )
       end
 
@@ -94,9 +102,11 @@ module KamalBackup
 
         result = bridge.execute_on_accessory(
           accessory_name: accessory_name,
-          command: Shellwords.join(argv)
+          command: Shellwords.join(argv),
+          stream: true
         )
-        print(result.stdout)
+        print(result.stdout) unless result.streamed
+        $stderr.print(result.stderr) if !result.streamed && !result.stderr.empty?
         result
       end
 
@@ -276,7 +286,7 @@ module KamalBackup
       desc "local [SNAPSHOT]", "Restore the backup into the local database and Active Storage path"
       def local(snapshot = "latest")
         confirm!("Restore #{snapshot} into the local database and Active Storage path? This will overwrite local data.")
-        puts(JSON.pretty_generate(local_app.restore_to_local_machine(snapshot)))
+        puts(JSON.pretty_generate(local_restore_app.restore_to_local_machine(snapshot)))
       end
 
       method_option :"confirm-production-restore", type: :boolean, default: false, desc: "Confirm production restore without interactive prompts"
@@ -301,9 +311,9 @@ module KamalBackup
       desc "local [SNAPSHOT]", "Run a restore drill on the local machine"
       def local(snapshot = "latest")
         confirm!("Run a local restore drill for #{snapshot}? This will overwrite local data.")
-        result = local_app.drill_on_local_machine(snapshot, check_command: options[:check])
+        result = local_restore_app.drill_on_local_machine(snapshot, check_command: options[:check])
         puts(JSON.pretty_generate(result))
-        exit(1) if local_app.drill_failed?(result)
+        exit(1) if local_restore_app.drill_failed?(result)
       end
 
       method_option :database, type: :string, desc: "Scratch database name for PostgreSQL or MySQL"
@@ -389,7 +399,9 @@ module KamalBackup
 
     def self.start(argv = ARGV, env: ENV)
       self.command_env = env
-      super(normalize_global_options(argv))
+      Command.with_output(CommandOutput.new(io: $stderr)) do
+        super(normalize_global_options(argv))
+      end
     rescue Error => e
       warn("kamal-backup: #{Redactor.new(env: env).redact_string(e.message)}")
       exit(1)

data/lib/kamal_backup/command.rb

@@ -1,4 +1,6 @@
 require "open3"
+require "securerandom"
+require "shellwords"
 require_relative "errors"
 
 module KamalBackup
@@ -18,14 +20,109 @@ module KamalBackup
 
     def display(redactor)
       env_prefix = env.keys.sort.map { |key| "#{key}=#{redactor.redact_value(key, env[key])}" }
-      redactor.redact_string((env_prefix + argv).join(" "))
+      redactor.redact_string((env_prefix + [argv.shelljoin]).join(" "))
     end
   end
 
-  CommandResult = Struct.new(:stdout, :stderr, :status, keyword_init: true)
+  CommandResult = Struct.new(:stdout, :stderr, :status, :streamed, keyword_init: true)
+
+  class CommandOutput
+    def initialize(io: $stdout)
+      @io = io
+      @mutex = Mutex.new
+      @buffers = {}
+    end
+
+    def info(message, redactor:)
+      write_line("  INFO #{redactor.redact_string(message)}")
+    end
+
+    def command_start(spec, redactor:)
+      id = SecureRandom.hex(4)
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      display = spec.display(redactor)
+
+      write_line("  INFO [#{id}] Running #{display} locally")
+      write_line(" DEBUG [#{id}] Command: #{display}")
+
+      { id: id, started_at: started_at, redactor: redactor }
+    end
+
+    def command_output(context, _stream, data, redactor:)
+      raw = data.to_s
+      return if raw.empty?
+
+      synchronize do
+        key = [context.fetch(:id), _stream]
+        @buffers[key] = "#{@buffers[key]}#{raw}"
+        flush_complete_output_lines(context, key, redactor: redactor)
+      end
+    end
+
+    def command_exit(context, status)
+      runtime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - context.fetch(:started_at)
+      result = status.to_i.zero? ? "successful" : "failed"
+
+      synchronize do
+        flush_output_buffers(context)
+        @io.puts("  INFO [#{context.fetch(:id)}] Finished in #{format("%.3f seconds", runtime)} with exit status #{status} (#{result}).")
+      end
+    end
+
+    private
+      def write_line(message)
+        synchronize { @io.puts(message) }
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
+      def flush_complete_output_lines(context, key, redactor:)
+        buffer = @buffers.fetch(key)
+        output = +""
+
+        while (index = buffer.index("\n"))
+          output << buffer.slice!(0..index)
+        end
+
+        @buffers[key] = buffer
+        write_output(context, output, redactor: redactor) unless output.empty?
+      end
+
+      def flush_output_buffers(context)
+        id = context.fetch(:id)
+        keys = @buffers.keys.select { |key_id, _stream| key_id == id }
+
+        keys.each do |key|
+          output = @buffers.delete(key)
+          next if output.to_s.empty?
+
+          write_output(context, output, redactor: context.fetch(:redactor))
+          @io.puts unless output.end_with?("\n")
+        end
+      end
+
+      def write_output(context, output, redactor:)
+        redactor.redact_string(output).each_line do |line|
+          @io.print(" DEBUG [#{context.fetch(:id)}] \t#{line}")
+        end
+        @io.flush if @io.respond_to?(:flush)
+      end
+  end
 
   class Command
     class << self
+      attr_accessor :output
+
+      def with_output(output)
+        previous_output = self.output
+        self.output = output
+        yield
+      ensure
+        self.output = previous_output
+      end
+
       def available?(name)
         ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).any? do |dir|
           path = File.join(dir, name)
@@ -33,9 +130,23 @@ module KamalBackup
         end
       end
 
-      def capture(spec, input: nil, redactor:)
-        stdout, stderr, status = Open3.capture3(spec.env, *spec.argv, stdin_data: input)
-        result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus)
+      def capture(spec, input: nil, redactor:, log: true, log_output: true, tee_stdout: nil, tee_stderr: nil)
+        output = log ? self.output : nil
+        return capture_quietly(spec, input: input, redactor: redactor) unless output || tee_stdout || tee_stderr
+
+        context = output&.command_start(spec, redactor: redactor)
+        stdout, stderr, status = popen_capture(
+          spec,
+          input: input,
+          redactor: redactor,
+          output: output,
+          context: context,
+          log_output: log_output,
+          tee_stdout: tee_stdout,
+          tee_stderr: tee_stderr
+        )
+        output&.command_exit(context, status.exitstatus)
+        result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus, streamed: !!(tee_stdout || tee_stderr))
 
         if status.success?
           result
@@ -46,7 +157,72 @@ module KamalBackup
         raise command_not_found(spec, e)
       end
 
+      def collect_stream(io, command_output: self.output, context: nil, stream: :stdout, log_output: true, tee_io: nil, redactor: nil)
+        captured_output = +""
+        tee_buffer = +""
+
+        loop do
+          chunk = io.readpartial(16 * 1024)
+          captured_output << chunk
+          command_output&.command_output(context, stream, chunk, redactor: redactor) if log_output && context
+          tee_buffer = tee_stream(tee_io, redactor, tee_buffer, chunk) if tee_io
+        rescue EOFError
+          flush_tee_stream(tee_io, redactor, tee_buffer) if tee_io
+          break
+        end
+
+        captured_output
+      end
+
       private
+        def tee_stream(io, redactor, buffer, chunk)
+          buffer << chunk
+
+          while (index = buffer.index("\n"))
+            io.print(redactor.redact_string(buffer.slice!(0..index)))
+          end
+
+          io.flush if io.respond_to?(:flush)
+          buffer
+        end
+
+        def flush_tee_stream(io, redactor, buffer)
+          return if buffer.empty?
+
+          io.print(redactor.redact_string(buffer))
+          io.flush if io.respond_to?(:flush)
+        end
+
+        def capture_quietly(spec, input:, redactor:)
+          stdout, stderr, status = Open3.capture3(spec.env, *spec.argv, stdin_data: input)
+          result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus, streamed: false)
+
+          if status.success?
+            result
+          else
+            raise command_failure(spec, status.exitstatus, stdout, stderr, redactor)
+          end
+        end
+
+        def popen_capture(spec, input:, redactor:, output:, context:, log_output:, tee_stdout:, tee_stderr:)
+          Open3.popen3(spec.env, *spec.argv) do |stdin, stdout, stderr, wait_thread|
+            stdin_writer = Thread.new do
+              stdin.write(input) if input
+            ensure
+              stdin.close unless stdin.closed?
+            end
+            stdout_reader = Thread.new do
+              collect_stream(stdout, command_output: output, context: context, stream: :stdout, log_output: log_output, tee_io: tee_stdout, redactor: redactor)
+            end
+            stderr_reader = Thread.new do
+              collect_stream(stderr, command_output: output, context: context, stream: :stderr, log_output: log_output, tee_io: tee_stderr, redactor: redactor)
+            end
+
+            stdin_writer.join
+            [stdout_reader.value, stderr_reader.value, wait_thread.value]
+          end
+        end
+
         def command_failure(spec, status, stdout, stderr, redactor)
           CommandError.new(
             "command failed (#{status}): #{spec.display(redactor)}\n#{redactor.redact_string(stderr)}",

data/lib/kamal_backup/kamal_bridge.rb

@@ -7,12 +7,14 @@ module KamalBackup
     DEFAULT_CONFIG_FILE = "config/deploy.yml"
     VERSION_LINE_PATTERN = /\A\d+(?:\.\d+)+(?:[-.][A-Za-z0-9]+)*\z/
 
-    def initialize(redactor:, config_file: nil, destination: nil, env: ENV, cwd: Dir.pwd)
+    def initialize(redactor:, config_file: nil, destination: nil, env: ENV, cwd: Dir.pwd, stdout: $stdout, stderr: $stderr)
       @redactor = redactor
       @config_file = config_file
       @destination = destination
       @env = env
       @cwd = cwd
+      @stdout = stdout
+      @stderr = stderr
     end
 
     def accessory_name(preferred: nil)
@@ -50,8 +52,8 @@ module KamalBackup
       accessory_secret_placeholders(accessory_name).merge(accessory_clear_env(accessory_name))
     end
 
-    def execute_on_accessory(accessory_name:, command:)
-      capture_kamal(kamal_exec_argv(accessory_name, command))
+    def execute_on_accessory(accessory_name:, command:, stream: false)
+      capture_kamal(kamal_exec_argv(accessory_name, command), stream: stream)
     end
 
     def remote_version(accessory_name:)
@@ -212,13 +214,19 @@ module KamalBackup
         argv
       end
 
-      def capture_kamal(argv)
+      def capture_kamal(argv, stream: false)
         spec = CommandSpec.new(argv: argv)
+        options = {
+          redactor: @redactor,
+          log_output: false,
+          tee_stdout: stream ? @stdout : nil,
+          tee_stderr: stream ? @stderr : nil
+        }
 
         if defined?(Bundler)
-          Bundler.with_unbundled_env { Command.capture(spec, redactor: @redactor) }
+          Bundler.with_unbundled_env { Command.capture(spec, **options) }
         else
-          Command.capture(spec, redactor: @redactor)
+          Command.capture(spec, **options)
         end
       end
 

data/lib/kamal_backup/restic.rb

@@ -16,7 +16,7 @@ module KamalBackup
     end
 
     def ensure_repository
-      run(%w[snapshots --json])
+      run(%w[snapshots --json], log_output: false)
     rescue CommandError => e
       if config.restic_init_if_missing?
         log("restic repository not ready, running restic init")
@@ -43,14 +43,17 @@ module KamalBackup
       log("backing up file content as #{filename}")
 
       File.open(path, "rb") do |file|
+        output = Command.output
+        context = output&.command_start(command, redactor: redactor)
         Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
-          stdout_reader = Thread.new { stdout.read }
-          stderr_reader = Thread.new { stderr.read }
+          stdout_reader = Thread.new { Command.collect_stream(stdout, command_output: output, context: context, stream: :stdout, redactor: redactor) }
+          stderr_reader = Thread.new { Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr, redactor: redactor) }
           IO.copy_stream(file, stdin)
           stdin.close
           out = stdout_reader.value
           err = stderr_reader.value
           status = wait_thread.value
+          output&.command_exit(context, status.exitstatus)
           raise_command_error(command, status, out, err) unless status.success?
 
           CommandResult.new(stdout: out, stderr: err, status: status.exitstatus)
@@ -99,7 +102,7 @@ module KamalBackup
     end
 
     def snapshots_json(tags: common_tags)
-      output = run(["snapshots", "--json"] + filter_tag_args(tags)).stdout
+      output = run(["snapshots", "--json"] + filter_tag_args(tags), log_output: false).stdout
       snapshots = JSON.parse(output)
       required_tags = tags.compact
       snapshots.select do |snapshot|
@@ -116,7 +119,7 @@ module KamalBackup
     end
 
     def ls_json(snapshot)
-      output = run(["ls", "--json", snapshot]).stdout
+      output = run(["ls", "--json", snapshot], log_output: false).stdout
       output.lines.filter_map do |line|
         JSON.parse(line)
       rescue JSON::ParserError
@@ -153,12 +156,15 @@ module KamalBackup
       FileUtils.mkdir_p(File.dirname(target_path))
       temp_path = "#{target_path}.kamal-backup-#{$$}.tmp"
 
+      output = Command.output
+      context = output&.command_start(command, redactor: redactor)
       Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
         stdin.close
-        stderr_reader = Thread.new { stderr.read }
+        stderr_reader = Thread.new { Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr, redactor: redactor) }
         File.open(temp_path, "wb") { |file| IO.copy_stream(stdout, file) }
         err = stderr_reader.value
         status = wait_thread.value
+        output&.command_exit(context, status.exitstatus)
         raise_command_error(command, status, "", err) unless status.success?
       end
       File.rename(temp_path, target_path)
@@ -176,8 +182,12 @@ module KamalBackup
       run(["restore", snapshot, "--target", target])
     end
 
-    def run(args)
-      Command.capture(CommandSpec.new(argv: ["restic"] + args, env: restic_env), redactor: redactor)
+    def run(args, log_output: true)
+      Command.capture(
+        CommandSpec.new(argv: ["restic"] + args, env: restic_env),
+        redactor: redactor,
+        log_output: log_output
+      )
     end
 
     def common_tags
@@ -240,13 +250,16 @@ module KamalBackup
       end
 
       def pipe_commands(producer, consumer, producer_label:, consumer_label:)
+        output = Command.output
+        producer_context = output&.command_start(producer, redactor: redactor)
         Open3.popen3(producer.env, *producer.argv) do |producer_stdin, producer_stdout, producer_stderr, producer_wait|
           producer_stdin.close
 
+          consumer_context = output&.command_start(consumer, redactor: redactor)
           Open3.popen3(consumer.env, *consumer.argv) do |consumer_stdin, consumer_stdout, consumer_stderr, consumer_wait|
-            producer_err_reader = Thread.new { producer_stderr.read }
-            consumer_out_reader = Thread.new { consumer_stdout.read }
-            consumer_err_reader = Thread.new { consumer_stderr.read }
+            producer_err_reader = Thread.new { Command.collect_stream(producer_stderr, command_output: output, context: producer_context, stream: :stderr, redactor: redactor) }
+            consumer_out_reader = Thread.new { Command.collect_stream(consumer_stdout, command_output: output, context: consumer_context, stream: :stdout, redactor: redactor) }
+            consumer_err_reader = Thread.new { Command.collect_stream(consumer_stderr, command_output: output, context: consumer_context, stream: :stderr, redactor: redactor) }
 
             copy_error = nil
             copy_thread = Thread.new do
@@ -260,6 +273,8 @@ module KamalBackup
             copy_thread.join
             producer_status = producer_wait.value
             consumer_status = consumer_wait.value
+            output&.command_exit(producer_context, producer_status.exitstatus)
+            output&.command_exit(consumer_context, consumer_status.exitstatus)
 
             producer_err = producer_err_reader.value
             consumer_out = consumer_out_reader.value
@@ -302,7 +317,11 @@ module KamalBackup
       end
 
       def log(message)
-        $stdout.puts("[kamal-backup] #{redactor.redact_string(message)}")
+        if Command.output
+          Command.output.info(message, redactor: redactor)
+        else
+          $stdout.puts("[kamal-backup] #{redactor.redact_string(message)}")
+        end
       end
   end
 end

data/lib/kamal_backup/version.rb

@@ -1,3 +1,3 @@
 module KamalBackup
-  VERSION = "0.3.0.beta5"
+  VERSION = "0.3.0.beta6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kamal-backup
 version: !ruby/object:Gem::Version
-  version: 0.3.0.beta5
+  version: 0.3.0.beta6
 platform: ruby
 authors:
 - crmne