kamal 2.10.1 → 2.12.0

data/lib/kamal/otel_shipper.rb

@@ -0,0 +1,176 @@
+require "active_support/core_ext/numeric/time"
+require "net/http"
+require "json"
+require "securerandom"
+require "uri"
+
+class Kamal::OtelShipper
+  BATCH_SIZE = 100
+  FLUSH_INTERVAL = 5.seconds
+
+  OTEL_ATTRIBUTE_KEYS = {
+    service: "service.namespace",
+    version: "kamal.deploy_version",
+    performer: "kamal.performer",
+    destination: "deployment.environment.name"
+  }
+
+  SEVERITIES = {
+    debug: { severityNumber: 5,  severityText: "DEBUG" },
+    info:  { severityNumber: 9,  severityText: "INFO" },
+    warn:  { severityNumber: 13, severityText: "WARN" },
+    error: { severityNumber: 17, severityText: "ERROR" },
+    fatal: { severityNumber: 21, severityText: "FATAL" }
+  }
+
+  LOGGER_SEVERITIES = {
+    Logger::DEBUG => :debug,
+    Logger::INFO  => :info,
+    Logger::WARN  => :warn,
+    Logger::ERROR => :error,
+    Logger::FATAL => :fatal
+  }
+
+  attr_reader :run_id
+
+  def initialize(endpoint:, tags:)
+    @endpoint = URI("#{endpoint.chomp('/')}/v1/logs")
+    @run_id = SecureRandom.uuid
+    @resource_attributes = [
+      { key: "service.name", value: { stringValue: "kamal" } },
+      { key: "service.version", value: { stringValue: Kamal::VERSION } },
+      { key: "kamal.run_id", value: { stringValue: @run_id } },
+      *tags.tags.map do |key, value|
+        otel_key = OTEL_ATTRIBUTE_KEYS.fetch(key, "kamal.#{key}")
+        { key: otel_key, value: { stringValue: value.to_s } }
+      end
+    ]
+    @buffer = Queue.new
+    @flush_mutex = Mutex.new
+    @running = true
+    @signal = Queue.new
+    @thread = start_flush_thread
+  end
+
+  def <<(str)
+    append(str)
+  end
+
+  def append(str, host: nil, iostream: nil, severity: nil)
+    otel_severity = LOGGER_SEVERITIES.fetch(severity, :info)
+    extra = build_context_attributes(host: host, iostream: iostream)
+    str.to_s.each_line do |line|
+      enqueue build_record(line.chomp, severity: otel_severity, attributes: extra)
+    end
+
+    self
+  end
+
+  def event(name, severity: :info, **attributes)
+    attrs = attributes.map { |k, v| { key: k.to_s, value: typed_value(v) } }
+    enqueue build_record(name, severity: severity, event_name: name, attributes: attrs)
+
+    self
+  end
+
+  def flush
+    @flush_mutex.synchronize do
+      lines = drain_buffer
+      ship(lines) if lines.any?
+    end
+  end
+
+  def shutdown
+    @running = false
+    @signal << true
+    @thread&.join(FLUSH_INTERVAL + 1.second)
+    flush
+  end
+
+  private
+    def enqueue(record)
+      @buffer << record
+      @signal << true if @buffer.size >= BATCH_SIZE
+    end
+
+    def start_flush_thread
+      Thread.new do
+        while @running
+          @signal.pop(timeout: FLUSH_INTERVAL)
+          flush
+        end
+      end
+    end
+
+    def drain_buffer
+      records = []
+      records << @buffer.pop(true) until @buffer.empty?
+      records
+    end
+
+    def ship(records)
+      with_connection do |http|
+        records.each_slice(BATCH_SIZE) do |batch|
+          ship_records(http, batch)
+        end
+      end
+    end
+
+    def build_record(body, severity: :info, event_name: nil, attributes: nil)
+      now = time_ns
+      { timeUnixNano: now, observedTimeUnixNano: now, **SEVERITIES.fetch(severity),
+        body: { stringValue: body }, eventName: event_name, attributes: attributes }.compact
+    end
+
+    def build_context_attributes(host:, iostream:)
+      attrs = []
+      attrs << { key: "server.address", value: { stringValue: host } } if host
+      attrs << { key: "log.iostream", value: { stringValue: iostream } } if iostream
+      attrs.presence
+    end
+
+    def typed_value(v)
+      case v
+      when Integer then { intValue: v }
+      when Float   then { doubleValue: v }
+      when Array   then { arrayValue: { values: v.map { |e| typed_value(e) } } }
+      else              { stringValue: v.to_s }
+      end
+    end
+
+    def with_connection
+      http = Net::HTTP.new(@endpoint.host, @endpoint.port)
+      http.use_ssl = @endpoint.scheme == "https"
+      http.open_timeout = 2.seconds
+      http.read_timeout = 5.seconds
+      http.start { |conn| yield conn }
+    rescue StandardError => e
+      unless @ship_error_logged
+        @ship_error_logged = true
+        $stderr.puts "OTel log shipping failed: #{e.class}: #{e.message}"
+        $stderr.puts e.backtrace.join("\n") if ENV["VERBOSE"]
+      end
+    end
+
+    def ship_records(http, records)
+      payload = {
+        resourceLogs: [ {
+          resource: { attributes: @resource_attributes },
+          scopeLogs: [ { scope: { name: "kamal", version: Kamal::VERSION }, logRecords: records } ]
+        } ]
+      }
+
+      req = Net::HTTP::Post.new(@endpoint.request_uri, "Content-Type" => "application/json")
+      req.body = JSON.generate(payload)
+      response = http.request(req)
+
+      unless response.is_a?(Net::HTTPSuccess) || @ship_error_logged
+        @ship_error_logged = true
+        $stderr.puts "OTel log shipping failed: HTTP #{response.code} #{response.message}"
+      end
+    end
+
+    def time_ns
+      Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond).to_s
+    end
+end

data/lib/kamal/output/base_logger.rb

@@ -0,0 +1,29 @@
+require "logger"
+
+class Kamal::Output::BaseLogger < ::Logger
+  def initialize
+    super(nil)
+    @subscription = ActiveSupport::Notifications.subscribe("modify.kamal", self)
+  end
+
+  def start(name, id, payload)
+    @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    on_start(payload)
+  end
+
+  def finish(name, id, payload)
+    runtime = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at).round(1)
+    on_finish(payload, runtime)
+  end
+
+  def add(severity, message = nil, progname = nil, &block)
+    if msg = message || (block ? block.call : progname)
+      self << msg.to_s
+    end
+  end
+
+  def close
+    ActiveSupport::Notifications.unsubscribe(@subscription)
+    on_close
+  end
+end

data/lib/kamal/output/file_logger.rb

@@ -0,0 +1,51 @@
+class Kamal::Output::FileLogger < Kamal::Output::BaseLogger
+  attr_reader :path
+
+  def self.build(settings:, config:)
+    raise ArgumentError, "file path is required" unless settings["path"]
+    new(path: settings["path"])
+  end
+
+  def initialize(path:)
+    @path = Pathname.new(path)
+    super()
+  end
+
+  def <<(message)
+    @file&.print(message)
+  end
+
+  private
+    def on_start(payload)
+      path.mkpath
+      @file_path = path.join(filename_for(payload))
+      @file = File.open(@file_path, "a")
+      @file.sync = true
+    end
+
+    def on_finish(payload, runtime)
+      if @file
+        if payload[:exception]
+          error_class, error_message = payload[:exception]
+          @file.puts "# FAILED: #{error_class}: #{error_message} (#{runtime}s)"
+        else
+          @file.puts "# Completed in #{runtime}s"
+        end
+        @file.close
+        @file = nil
+        puts "Logs written to #{@file_path}"
+      end
+    end
+
+    def on_close
+      if @file
+        @file.close
+        @file = nil
+      end
+    end
+
+    def filename_for(payload)
+      command = [ payload[:command], payload[:subcommand] ].compact.join("_")
+      [ Time.now.strftime("%Y-%m-%dT%H-%M-%S"), payload[:destination], command ].compact.join("_") + ".log"
+    end
+end

data/lib/kamal/output/formatter.rb

@@ -0,0 +1,36 @@
+class Kamal::Output::Formatter < SSHKit::Formatter::Pretty
+  def initialize(output, logger)
+    @logger = logger
+    super(output)
+  end
+
+  def log_command_start(command)
+    with_command_context(command) { super }
+  end
+
+  def log_command_data(command, stream_type, stream_data)
+    with_command_context(command, iostream: stream_type.to_s) { super }
+  end
+
+  def log_command_exit(command)
+    with_command_context(command) { super }
+  end
+
+  private
+    def write_message(verbosity, message, uuid = nil)
+      super
+      Thread.current[:kamal_severity] = verbosity
+      @logger << "#{format_message(verbosity, message, uuid)}\n" rescue nil
+    ensure
+      Thread.current[:kamal_severity] = nil
+    end
+
+    def with_command_context(command, iostream: nil)
+      Thread.current[:kamal_host] = command.host.to_s
+      Thread.current[:kamal_iostream] = iostream
+      yield
+    ensure
+      Thread.current[:kamal_host] = nil
+      Thread.current[:kamal_iostream] = nil
+    end
+end

data/lib/kamal/output/otel_logger.rb

@@ -0,0 +1,70 @@
+class Kamal::Output::OtelLogger < Kamal::Output::BaseLogger
+  def self.build(settings:, config:)
+    raise ArgumentError, "OTel endpoint is required" unless settings["endpoint"]
+    new(
+      endpoint: settings["endpoint"],
+      tags: Kamal::Tags.from_config(config).except(:service_version, :recorded_at),
+      service: config.service
+    )
+  end
+
+  def initialize(endpoint:, tags:, service: nil)
+    @endpoint = endpoint
+    @shipper = Kamal::OtelShipper.new(endpoint: endpoint, tags: tags)
+    @service = service
+    super()
+  end
+
+  def <<(message)
+    host = Thread.current[:kamal_host]
+    iostream = Thread.current[:kamal_iostream]
+    severity = Thread.current[:kamal_severity]
+    @shipper.append(message, host: host, iostream: iostream, severity: severity)
+  end
+
+  DEPLOY_COMMANDS = %w[ deploy redeploy rollback setup ].freeze
+
+  private
+    def on_start(payload)
+      @shipper.event("kamal.start",
+        "kamal.command": full_command(payload),
+        **deployment_attrs(payload))
+    end
+
+    def on_finish(payload, runtime)
+      if payload[:exception]
+        error_class, error_message = payload[:exception]
+        @shipper.event("kamal.failed", severity: :error,
+          "kamal.command": full_command(payload), "kamal.runtime": runtime,
+          "exception.type": error_class, "exception.message": error_message,
+          **deployment_attrs(payload, status: "failed"))
+      else
+        @shipper.event("kamal.complete",
+          "kamal.command": full_command(payload), "kamal.runtime": runtime,
+          **deployment_attrs(payload, status: "succeeded"))
+      end
+      puts "Logs sent to #{@endpoint}"
+    end
+
+    def on_close
+      @shipper.shutdown
+    end
+
+    def full_command(payload)
+      [ payload[:command], payload[:subcommand] ].compact.join(" ")
+    end
+
+    def deploy?(payload)
+      DEPLOY_COMMANDS.include?(payload[:command])
+    end
+
+    def deployment_attrs(payload, status: nil)
+      if deploy?(payload)
+        attrs = { "deployment.id": @shipper.run_id, "deployment.name": "#{full_command(payload)} #{@service}" }
+        attrs[:"deployment.status"] = status if status
+        attrs
+      else
+        {}
+      end
+    end
+end

data/lib/kamal/secrets/adapters/aws_secrets_manager.rb

@@ -14,8 +14,12 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba
           secret_name = secret["Name"]
           secret_string = JSON.parse(secret["SecretString"])
 
-          secret_string.each do |key, value|
-            results["#{secret_name}/#{key}"] = value
+          if secret_string.is_a?(Hash)
+            secret_string.each do |key, value|
+              results["#{secret_name}/#{key}"] = stringify_secret_value(value)
+            end
+          else
+            results["#{secret_name}"] = stringify_secret_value(secret_string)
           end
         rescue JSON::ParserError
           results["#{secret_name}"] = secret["SecretString"]
@@ -40,6 +44,10 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba
       end
     end
 
+    def stringify_secret_value(value)
+      value.is_a?(String) ? value : JSON.dump(value)
+    end
+
     def check_dependencies!
       raise RuntimeError, "AWS CLI is not installed" unless cli_installed?
     end

data/lib/kamal/secrets/adapters/passbolt.rb

@@ -47,7 +47,7 @@ class Kamal::Secrets::Adapters::Passbolt < Kamal::Secrets::Adapters::Base
       end
 
       filter_condition = filter_conditions.any? ? "--filter '#{filter_conditions.join(" || ")}'" : ""
-      items = `passbolt list resources #{filter_condition} #{folders.map { |item| "--folder #{item["id"].to_s.shellescape}" }.join(" ")} --json`
+      items = `passbolt list resources #{filter_condition} #{folders.map { |item| "--folder #{item["id"].to_s.shellescape}" }.join(" ")} --column name --column password --json`
       raise RuntimeError, "Could not read #{secrets} from Passbolt" unless $?.success?
       items = JSON.parse(items)
       found_names = items.map { |item| item["name"] }

data/lib/kamal/secrets.rb

@@ -3,7 +3,7 @@ require "dotenv"
 class Kamal::Secrets
   Kamal::Secrets::Dotenv::InlineCommandSubstitution.install!
 
-  def initialize(destination: nil, secrets_path:)
+  def initialize(destination: nil, secrets_path: ".kamal/secrets")
     @destination = destination
     @secrets_path = secrets_path
     @mutex = Mutex.new

data/lib/kamal/sshkit_with_ext.rb

@@ -19,11 +19,16 @@ class SSHKit::Backend::Abstract
     JSON.pretty_generate(JSON.parse(capture(*args, **kwargs)))
   end
 
-  def puts_by_host(host, output, type: "App", quiet: false)
-    unless quiet
-      puts "#{type} Host: #{host}"
+  def puts_by_host(host, output, type: "App", quiet: false, raw: false)
+    if raw
+      $stdout.binmode
+      $stdout.write(output)
+    else
+      unless quiet
+        puts "#{type} Host: #{host}"
+      end
+      puts "#{output}\n\n"
     end
-    puts "#{output}\n\n"
   end
 
   # Our execution pattern is for the CLI execute args lists returned

data/lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "2.10.1"
+  VERSION = "2.12.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kamal
 version: !ruby/object:Gem::Version
-  version: 2.10.1
+  version: 2.12.0
 platform: ruby
 authors:
 - David Heinemeier Hansson
@@ -175,6 +175,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -288,6 +302,7 @@ files:
 - lib/kamal/configuration/docs/configuration.yml
 - lib/kamal/configuration/docs/env.yml
 - lib/kamal/configuration/docs/logging.yml
+- lib/kamal/configuration/docs/output.yml
 - lib/kamal/configuration/docs/proxy.yml
 - lib/kamal/configuration/docs/registry.yml
 - lib/kamal/configuration/docs/role.yml
@@ -297,6 +312,7 @@ files:
 - lib/kamal/configuration/env.rb
 - lib/kamal/configuration/env/tag.rb
 - lib/kamal/configuration/logging.rb
+- lib/kamal/configuration/output.rb
 - lib/kamal/configuration/proxy.rb
 - lib/kamal/configuration/proxy/boot.rb
 - lib/kamal/configuration/proxy/run.rb
@@ -320,6 +336,11 @@ files:
 - lib/kamal/docker.rb
 - lib/kamal/env_file.rb
 - lib/kamal/git.rb
+- lib/kamal/otel_shipper.rb
+- lib/kamal/output/base_logger.rb
+- lib/kamal/output/file_logger.rb
+- lib/kamal/output/formatter.rb
+- lib/kamal/output/otel_logger.rb
 - lib/kamal/secrets.rb
 - lib/kamal/secrets/adapters.rb
 - lib/kamal/secrets/adapters/aws_secrets_manager.rb
@@ -357,7 +378,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Deploy web apps in containers to servers running Docker with zero downtime.
 test_files: []