dash 2.12.0

data/lib/kamal/otel_shipper.rb

@@ -0,0 +1,176 @@
+require "active_support/core_ext/numeric/time"
+require "net/http"
+require "json"
+require "securerandom"
+require "uri"
+
+class Kamal::OtelShipper
+  BATCH_SIZE = 100
+  FLUSH_INTERVAL = 5.seconds
+
+  OTEL_ATTRIBUTE_KEYS = {
+    service: "service.namespace",
+    version: "kamal.deploy_version",
+    performer: "kamal.performer",
+    destination: "deployment.environment.name"
+  }
+
+  SEVERITIES = {
+    debug: { severityNumber: 5,  severityText: "DEBUG" },
+    info:  { severityNumber: 9,  severityText: "INFO" },
+    warn:  { severityNumber: 13, severityText: "WARN" },
+    error: { severityNumber: 17, severityText: "ERROR" },
+    fatal: { severityNumber: 21, severityText: "FATAL" }
+  }
+
+  LOGGER_SEVERITIES = {
+    Logger::DEBUG => :debug,
+    Logger::INFO  => :info,
+    Logger::WARN  => :warn,
+    Logger::ERROR => :error,
+    Logger::FATAL => :fatal
+  }
+
+  attr_reader :run_id
+
+  def initialize(endpoint:, tags:)
+    @endpoint = URI("#{endpoint.chomp('/')}/v1/logs")
+    @run_id = SecureRandom.uuid
+    @resource_attributes = [
+      { key: "service.name", value: { stringValue: "kamal" } },
+      { key: "service.version", value: { stringValue: Kamal::VERSION } },
+      { key: "kamal.run_id", value: { stringValue: @run_id } },
+      *tags.tags.map do |key, value|
+        otel_key = OTEL_ATTRIBUTE_KEYS.fetch(key, "kamal.#{key}")
+        { key: otel_key, value: { stringValue: value.to_s } }
+      end
+    ]
+    @buffer = Queue.new
+    @flush_mutex = Mutex.new
+    @running = true
+    @signal = Queue.new
+    @thread = start_flush_thread
+  end
+
+  def <<(str)
+    append(str)
+  end
+
+  def append(str, host: nil, iostream: nil, severity: nil)
+    otel_severity = LOGGER_SEVERITIES.fetch(severity, :info)
+    extra = build_context_attributes(host: host, iostream: iostream)
+    str.to_s.each_line do |line|
+      enqueue build_record(line.chomp, severity: otel_severity, attributes: extra)
+    end
+
+    self
+  end
+
+  def event(name, severity: :info, **attributes)
+    attrs = attributes.map { |k, v| { key: k.to_s, value: typed_value(v) } }
+    enqueue build_record(name, severity: severity, event_name: name, attributes: attrs)
+
+    self
+  end
+
+  def flush
+    @flush_mutex.synchronize do
+      lines = drain_buffer
+      ship(lines) if lines.any?
+    end
+  end
+
+  def shutdown
+    @running = false
+    @signal << true
+    @thread&.join(FLUSH_INTERVAL + 1.second)
+    flush
+  end
+
+  private
+    def enqueue(record)
+      @buffer << record
+      @signal << true if @buffer.size >= BATCH_SIZE
+    end
+
+    def start_flush_thread
+      Thread.new do
+        while @running
+          @signal.pop(timeout: FLUSH_INTERVAL)
+          flush
+        end
+      end
+    end
+
+    def drain_buffer
+      records = []
+      records << @buffer.pop(true) until @buffer.empty?
+      records
+    end
+
+    def ship(records)
+      with_connection do |http|
+        records.each_slice(BATCH_SIZE) do |batch|
+          ship_records(http, batch)
+        end
+      end
+    end
+
+    def build_record(body, severity: :info, event_name: nil, attributes: nil)
+      now = time_ns
+      { timeUnixNano: now, observedTimeUnixNano: now, **SEVERITIES.fetch(severity),
+        body: { stringValue: body }, eventName: event_name, attributes: attributes }.compact
+    end
+
+    def build_context_attributes(host:, iostream:)
+      attrs = []
+      attrs << { key: "server.address", value: { stringValue: host } } if host
+      attrs << { key: "log.iostream", value: { stringValue: iostream } } if iostream
+      attrs.presence
+    end
+
+    def typed_value(v)
+      case v
+      when Integer then { intValue: v }
+      when Float   then { doubleValue: v }
+      when Array   then { arrayValue: { values: v.map { |e| typed_value(e) } } }
+      else              { stringValue: v.to_s }
+      end
+    end
+
+    def with_connection
+      http = Net::HTTP.new(@endpoint.host, @endpoint.port)
+      http.use_ssl = @endpoint.scheme == "https"
+      http.open_timeout = 2.seconds
+      http.read_timeout = 5.seconds
+      http.start { |conn| yield conn }
+    rescue StandardError => e
+      unless @ship_error_logged
+        @ship_error_logged = true
+        $stderr.puts "OTel log shipping failed: #{e.class}: #{e.message}"
+        $stderr.puts e.backtrace.join("\n") if ENV["VERBOSE"]
+      end
+    end
+
+    def ship_records(http, records)
+      payload = {
+        resourceLogs: [ {
+          resource: { attributes: @resource_attributes },
+          scopeLogs: [ { scope: { name: "kamal", version: Kamal::VERSION }, logRecords: records } ]
+        } ]
+      }
+
+      req = Net::HTTP::Post.new(@endpoint.request_uri, "Content-Type" => "application/json")
+      req.body = JSON.generate(payload)
+      response = http.request(req)
+
+      unless response.is_a?(Net::HTTPSuccess) || @ship_error_logged
+        @ship_error_logged = true
+        $stderr.puts "OTel log shipping failed: HTTP #{response.code} #{response.message}"
+      end
+    end
+
+    def time_ns
+      Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond).to_s
+    end
+end

data/lib/kamal/output/base_logger.rb

@@ -0,0 +1,29 @@
+require "logger"
+
+class Kamal::Output::BaseLogger < ::Logger
+  def initialize
+    super(nil)
+    @subscription = ActiveSupport::Notifications.subscribe("modify.kamal", self)
+  end
+
+  def start(name, id, payload)
+    @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    on_start(payload)
+  end
+
+  def finish(name, id, payload)
+    runtime = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at).round(1)
+    on_finish(payload, runtime)
+  end
+
+  def add(severity, message = nil, progname = nil, &block)
+    if msg = message || (block ? block.call : progname)
+      self << msg.to_s
+    end
+  end
+
+  def close
+    ActiveSupport::Notifications.unsubscribe(@subscription)
+    on_close
+  end
+end

data/lib/kamal/output/file_logger.rb

@@ -0,0 +1,51 @@
+class Kamal::Output::FileLogger < Kamal::Output::BaseLogger
+  attr_reader :path
+
+  def self.build(settings:, config:)
+    raise ArgumentError, "file path is required" unless settings["path"]
+    new(path: settings["path"])
+  end
+
+  def initialize(path:)
+    @path = Pathname.new(path)
+    super()
+  end
+
+  def <<(message)
+    @file&.print(message)
+  end
+
+  private
+    def on_start(payload)
+      path.mkpath
+      @file_path = path.join(filename_for(payload))
+      @file = File.open(@file_path, "a")
+      @file.sync = true
+    end
+
+    def on_finish(payload, runtime)
+      if @file
+        if payload[:exception]
+          error_class, error_message = payload[:exception]
+          @file.puts "# FAILED: #{error_class}: #{error_message} (#{runtime}s)"
+        else
+          @file.puts "# Completed in #{runtime}s"
+        end
+        @file.close
+        @file = nil
+        puts "Logs written to #{@file_path}"
+      end
+    end
+
+    def on_close
+      if @file
+        @file.close
+        @file = nil
+      end
+    end
+
+    def filename_for(payload)
+      command = [ payload[:command], payload[:subcommand] ].compact.join("_")
+      [ Time.now.strftime("%Y-%m-%dT%H-%M-%S"), payload[:destination], command ].compact.join("_") + ".log"
+    end
+end

data/lib/kamal/output/formatter.rb

@@ -0,0 +1,36 @@
+class Kamal::Output::Formatter < SSHKit::Formatter::Pretty
+  def initialize(output, logger)
+    @logger = logger
+    super(output)
+  end
+
+  def log_command_start(command)
+    with_command_context(command) { super }
+  end
+
+  def log_command_data(command, stream_type, stream_data)
+    with_command_context(command, iostream: stream_type.to_s) { super }
+  end
+
+  def log_command_exit(command)
+    with_command_context(command) { super }
+  end
+
+  private
+    def write_message(verbosity, message, uuid = nil)
+      super
+      Thread.current[:kamal_severity] = verbosity
+      @logger << "#{format_message(verbosity, message, uuid)}\n" rescue nil
+    ensure
+      Thread.current[:kamal_severity] = nil
+    end
+
+    def with_command_context(command, iostream: nil)
+      Thread.current[:kamal_host] = command.host.to_s
+      Thread.current[:kamal_iostream] = iostream
+      yield
+    ensure
+      Thread.current[:kamal_host] = nil
+      Thread.current[:kamal_iostream] = nil
+    end
+end

data/lib/kamal/output/otel_logger.rb

@@ -0,0 +1,70 @@
+class Kamal::Output::OtelLogger < Kamal::Output::BaseLogger
+  def self.build(settings:, config:)
+    raise ArgumentError, "OTel endpoint is required" unless settings["endpoint"]
+    new(
+      endpoint: settings["endpoint"],
+      tags: Kamal::Tags.from_config(config).except(:service_version, :recorded_at),
+      service: config.service
+    )
+  end
+
+  def initialize(endpoint:, tags:, service: nil)
+    @endpoint = endpoint
+    @shipper = Kamal::OtelShipper.new(endpoint: endpoint, tags: tags)
+    @service = service
+    super()
+  end
+
+  def <<(message)
+    host = Thread.current[:kamal_host]
+    iostream = Thread.current[:kamal_iostream]
+    severity = Thread.current[:kamal_severity]
+    @shipper.append(message, host: host, iostream: iostream, severity: severity)
+  end
+
+  DEPLOY_COMMANDS = %w[ deploy redeploy rollback setup ].freeze
+
+  private
+    def on_start(payload)
+      @shipper.event("kamal.start",
+        "kamal.command": full_command(payload),
+        **deployment_attrs(payload))
+    end
+
+    def on_finish(payload, runtime)
+      if payload[:exception]
+        error_class, error_message = payload[:exception]
+        @shipper.event("kamal.failed", severity: :error,
+          "kamal.command": full_command(payload), "kamal.runtime": runtime,
+          "exception.type": error_class, "exception.message": error_message,
+          **deployment_attrs(payload, status: "failed"))
+      else
+        @shipper.event("kamal.complete",
+          "kamal.command": full_command(payload), "kamal.runtime": runtime,
+          **deployment_attrs(payload, status: "succeeded"))
+      end
+      puts "Logs sent to #{@endpoint}"
+    end
+
+    def on_close
+      @shipper.shutdown
+    end
+
+    def full_command(payload)
+      [ payload[:command], payload[:subcommand] ].compact.join(" ")
+    end
+
+    def deploy?(payload)
+      DEPLOY_COMMANDS.include?(payload[:command])
+    end
+
+    def deployment_attrs(payload, status: nil)
+      if deploy?(payload)
+        attrs = { "deployment.id": @shipper.run_id, "deployment.name": "#{full_command(payload)} #{@service}" }
+        attrs[:"deployment.status"] = status if status
+        attrs
+      else
+        {}
+      end
+    end
+end

data/lib/kamal/secrets/adapters/aws_secrets_manager.rb

@@ -0,0 +1,59 @@
+class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    def login(_account)
+      nil
+    end
+
+    def fetch_secrets(secrets, from:, account: nil, session:)
+      {}.tap do |results|
+        get_from_secrets_manager(prefixed_secrets(secrets, from: from), account: account).each do |secret|
+          secret_name = secret["Name"]
+          secret_string = JSON.parse(secret["SecretString"])
+
+          if secret_string.is_a?(Hash)
+            secret_string.each do |key, value|
+              results["#{secret_name}/#{key}"] = stringify_secret_value(value)
+            end
+          else
+            results["#{secret_name}"] = stringify_secret_value(secret_string)
+          end
+        rescue JSON::ParserError
+          results["#{secret_name}"] = secret["SecretString"]
+        end
+      end
+    end
+
+    def get_from_secrets_manager(secrets, account: nil)
+      args = [ "aws", "secretsmanager", "batch-get-secret-value", "--secret-id-list" ] + secrets.map(&:shellescape)
+      args += [ "--profile", account.shellescape ] if account
+      args += [ "--output", "json" ]
+      cmd = args.join(" ")
+
+      `#{cmd}`.tap do |secrets|
+        raise RuntimeError, "Could not read #{secrets} from AWS Secrets Manager" unless $?.success?
+
+        secrets = JSON.parse(secrets)
+
+        return secrets["SecretValues"] unless secrets["Errors"].present?
+
+        raise RuntimeError, secrets["Errors"].map { |error| "#{error['SecretId']}: #{error['Message']}" }.join(" ")
+      end
+    end
+
+    def stringify_secret_value(value)
+      value.is_a?(String) ? value : JSON.dump(value)
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "AWS CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `aws --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/base.rb

@@ -0,0 +1,33 @@
+class Kamal::Secrets::Adapters::Base
+  delegate :optionize, to: Kamal::Utils
+
+  def fetch(secrets, account: nil, from: nil)
+    raise RuntimeError, "Missing required option '--account'" if requires_account? && account.blank?
+
+    check_dependencies!
+
+    session = login(account)
+    fetch_secrets(secrets, from: from, account: account, session: session)
+  end
+
+  def requires_account?
+    true
+  end
+
+  private
+    def login(...)
+      raise NotImplementedError
+    end
+
+    def fetch_secrets(...)
+      raise NotImplementedError
+    end
+
+    def check_dependencies!
+      raise NotImplementedError
+    end
+
+    def prefixed_secrets(secrets, from:)
+      secrets.map { |secret| [ from, secret ].compact.join("/") }
+    end
+end

data/lib/kamal/secrets/adapters/bitwarden.rb

@@ -0,0 +1,81 @@
+class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base
+  private
+    def login(account)
+      status = run_command("status")
+
+      if status["status"] == "unauthenticated"
+        run_command("login #{account.shellescape}", raw: true)
+        status = run_command("status")
+      end
+
+      if status["status"] == "locked"
+        session = run_command("unlock --raw", raw: true).presence
+        status = run_command("status", session: session)
+      end
+
+      raise RuntimeError, "Failed to login to and unlock Bitwarden" unless status["status"] == "unlocked"
+
+      run_command("sync", session: session, raw: true)
+      raise RuntimeError, "Failed to sync Bitwarden" unless $?.success?
+
+      session
+    end
+
+    def fetch_secrets(secrets, from:, account:, session:)
+      {}.tap do |results|
+        items_fields(prefixed_secrets(secrets, from: from)).each do |item, fields|
+          item_json = run_command("get item #{item.shellescape}", session: session, raw: true)
+          raise RuntimeError, "Could not read #{item} from Bitwarden" unless $?.success?
+          item_json = JSON.parse(item_json)
+          if fields.any?
+            results.merge! fetch_secrets_from_fields(fields, item, item_json)
+          elsif item_json.dig("login", "password")
+            results[item] = item_json.dig("login", "password")
+          elsif item_json["fields"]&.any?
+            fields = item_json["fields"].pluck("name")
+            results.merge! fetch_secrets_from_fields(fields, item, item_json)
+          else
+            raise RuntimeError, "Item #{item} is not a login type item and no fields were specified"
+          end
+        end
+      end
+    end
+
+    def fetch_secrets_from_fields(fields, item, item_json)
+      fields.to_h do |field|
+        item_field = item_json["fields"].find { |f| f["name"] == field }
+        raise RuntimeError, "Could not find field #{field} in item #{item} in Bitwarden" unless item_field
+        value = item_field["value"]
+        [ "#{item}/#{field}", value ]
+      end
+    end
+
+    def items_fields(secrets)
+      {}.tap do |items|
+        secrets.each do |secret|
+          item, field = secret.split("/")
+          items[item] ||= []
+          items[item] << field
+        end
+      end
+    end
+
+    def signedin?(account)
+      run_command("status")["status"] != "unauthenticated"
+    end
+
+    def run_command(command, session: nil, raw: false)
+      full_command = [ *("BW_SESSION=#{session.shellescape}" if session), "bw", command ].join(" ")
+      result = `#{full_command}`.strip
+      raw ? result : JSON.parse(result)
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Bitwarden CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `bw --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb

@@ -0,0 +1,66 @@
+class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    LIST_ALL_SELECTOR = "all"
+    LIST_ALL_FROM_PROJECT_SUFFIX = "/all"
+    LIST_COMMAND = "secret list"
+    GET_COMMAND = "secret get"
+
+    def fetch_secrets(secrets, from:, account:, session:)
+      raise RuntimeError, "You must specify what to retrieve from Bitwarden Secrets Manager" if secrets.length == 0
+
+      secrets = prefixed_secrets(secrets, from: from)
+      command, project = extract_command_and_project(secrets)
+
+      {}.tap do |results|
+        if command.nil?
+          secrets.each do |secret_uuid|
+            item_json = run_command("#{GET_COMMAND} #{secret_uuid.shellescape}")
+            raise RuntimeError, "Could not read #{secret_uuid} from Bitwarden Secrets Manager" unless $?.success?
+            item_json = JSON.parse(item_json)
+            results[item_json["key"]] = item_json["value"]
+          end
+        else
+          items_json = run_command(command)
+          raise RuntimeError, "Could not read secrets from Bitwarden Secrets Manager" unless $?.success?
+
+          JSON.parse(items_json).each do |item_json|
+            results[item_json["key"]] = item_json["value"]
+          end
+        end
+      end
+    end
+
+    def extract_command_and_project(secrets)
+      if secrets.length == 1
+        if secrets[0] == LIST_ALL_SELECTOR
+          [ LIST_COMMAND, nil ]
+        elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX)
+          project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first
+          [ "#{LIST_COMMAND} #{project.shellescape}", project ]
+        end
+      end
+    end
+
+    def run_command(command, session: nil)
+      full_command = [ "bws", command ].join(" ")
+      `#{full_command}`
+    end
+
+    def login(account)
+      run_command("project list")
+      raise RuntimeError, "Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?" unless $?.success?
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Bitwarden Secrets Manager CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `bws --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/doppler.rb

@@ -0,0 +1,57 @@
+class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    def login(*)
+      unless loggedin?
+        `doppler login -y`
+        raise RuntimeError, "Failed to login to Doppler" unless $?.success?
+      end
+    end
+
+    def loggedin?
+      `doppler me --json 2> /dev/null`
+      $?.success?
+    end
+
+    def fetch_secrets(secrets, from:, **)
+      secrets = prefixed_secrets(secrets, from: from)
+      flags = secrets_get_flags(secrets)
+
+      secret_names = secrets.collect { |s| s.split("/").last }
+
+      items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{flags}`
+      raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success?
+
+      items = JSON.parse(items)
+
+      items.transform_values { |value| value["computed"] }
+    end
+
+    def secrets_get_flags(secrets)
+      unless service_token_set?
+        project, config, _ = secrets.first.split("/")
+
+        unless project && config
+          raise RuntimeError, "Missing project or config from '--from=project/config' option"
+        end
+
+        project_and_config_flags = "-p #{project.shellescape} -c #{config.shellescape}"
+      end
+    end
+
+    def service_token_set?
+      ENV["DOPPLER_TOKEN"] && ENV["DOPPLER_TOKEN"][0, 5] == "dp.st"
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Doppler CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `doppler --version 2> /dev/null`
+      $?.success?
+    end
+end