kamal 1.8.3 → 2.7.0

data/lib/kamal/configuration.rb

@@ -2,21 +2,22 @@ require "active_support/ordered_options"
 require "active_support/core_ext/string/inquiry"
 require "active_support/core_ext/module/delegation"
 require "active_support/core_ext/hash/keys"
-require "pathname"
 require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :labels, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
+  delegate :service, :image, :labels, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_reader :destination, :raw_config
-  attr_reader :accessories, :boot, :builder, :env, :healthcheck, :logging, :traefik, :servers, :ssh, :sshkit, :registry
+  attr_reader :destination, :raw_config, :secrets
+  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :proxy, :proxy_boot, :servers, :ssh, :sshkit, :registry
 
   include Validation
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
+      ENV["KAMAL_DESTINATION"] = destination
+
       raw_config = load_config_files(config_file, *destination_config_file(config_file, destination))
 
       new raw_config, destination: destination, version: version
@@ -31,7 +32,7 @@ class Kamal::Configuration
         if file.exist?
           # Newer Psych doesn't load aliases by default
           load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load
-          YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys
+          YAML.send(load_method, ERB.new(File.read(file)).result).symbolize_keys
         else
           raise "Configuration file not found in #{file}"
         end
@@ -49,18 +50,21 @@ class Kamal::Configuration
 
     validate! raw_config, example: validation_yml.symbolize_keys, context: "", with: Kamal::Configuration::Validator::Configuration
 
+    @secrets = Kamal::Secrets.new(destination: destination)
+
     # Eager load config to validate it, these are first as they have dependencies later on
     @servers = Servers.new(config: self)
-    @registry = Registry.new(config: self)
+    @registry = Registry.new(config: @raw_config, secrets: secrets)
 
     @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || []
+    @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
     @boot = Boot.new(config: self)
     @builder = Builder.new(config: self)
-    @env = Env.new(config: @raw_config.env || {})
+    @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
 
-    @healthcheck = Healthcheck.new(healthcheck_config: @raw_config.healthcheck)
     @logging = Logging.new(logging_config: @raw_config.logging)
-    @traefik = Traefik.new(config: self)
+    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy, secrets: secrets)
+    @proxy_boot = Proxy::Boot.new(config: self)
     @ssh = Ssh.new(config: self)
     @sshkit = Sshkit.new(config: self)
 
@@ -69,9 +73,11 @@ class Kamal::Configuration
     ensure_valid_kamal_version
     ensure_retain_containers_valid
     ensure_valid_service_name
+    ensure_no_traefik_reboot_hooks
+    ensure_one_host_for_ssl_roles
+    ensure_unique_hosts_for_ssl_roles
   end
 
-
   def version=(version)
     @declared_version = version
   end
@@ -95,6 +101,9 @@ class Kamal::Configuration
     raw_config.minimum_version
   end
 
+  def service_and_destination
+    [ service, destination ].compact.join("-")
+  end
 
   def roles
     servers.roles
@@ -108,11 +117,14 @@ class Kamal::Configuration
     accessories.detect { |a| a.name == name.to_s }
   end
 
-
   def all_hosts
     (roles + accessories).flat_map(&:hosts).uniq
   end
 
+  def app_hosts
+    roles.flat_map(&:hosts).uniq
+  end
+
   def primary_host
     primary_role&.primary_host
   end
@@ -129,16 +141,20 @@ class Kamal::Configuration
     raw_config.allow_empty_roles
   end
 
-  def traefik_roles
-    roles.select(&:running_traefik?)
+  def proxy_roles
+    roles.select(&:running_proxy?)
+  end
+
+  def proxy_role_names
+    proxy_roles.flat_map(&:name)
   end
 
-  def traefik_role_names
-    traefik_roles.flat_map(&:name)
+  def proxy_accessories
+    accessories.select(&:running_proxy?)
   end
 
-  def traefik_hosts
-    traefik_roles.flat_map(&:hosts).uniq
+  def proxy_hosts
+    (proxy_roles.flat_map(&:hosts) + proxy_accessories.flat_map(&:hosts)).uniq
   end
 
   def repository
@@ -169,7 +185,6 @@ class Kamal::Configuration
     raw_config.retain_containers || 5
   end
 
-
   def volume_args
     if raw_config.volumes.present?
       argumentize "--volume", raw_config.volumes
@@ -182,30 +197,36 @@ class Kamal::Configuration
     logging.args
   end
 
-
-  def healthcheck_service
-    [ "healthcheck", service, destination ].compact.join("-")
-  end
-
   def readiness_delay
     raw_config.readiness_delay || 7
   end
 
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
+  def deploy_timeout
+    raw_config.deploy_timeout || 30
   end
 
+  def drain_timeout
+    raw_config.drain_timeout || 30
+  end
 
   def run_directory
-    raw_config.run_directory || ".kamal"
+    ".kamal"
   end
 
-  def run_directory_as_docker_volume
-    if Pathname.new(run_directory).absolute?
-      run_directory
-    else
-      File.join "$(pwd)", run_directory
-    end
+  def apps_directory
+    File.join run_directory, "apps"
+  end
+
+  def app_directory
+    File.join apps_directory, service_and_destination
+  end
+
+  def env_directory
+    File.join app_directory, "env"
+  end
+
+  def assets_directory
+    File.join app_directory, "assets"
   end
 
   def hooks_path
@@ -216,14 +237,13 @@ class Kamal::Configuration
     raw_config.asset_path
   end
 
-
-  def host_env_directory
-    File.join(run_directory, "env")
+  def error_pages_path
+    raw_config.error_pages_path
   end
 
   def env_tags
     @env_tags ||= if (tags = raw_config.env["tags"])
-      tags.collect { |name, config| Env::Tag.new(name, config: config) }
+      tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
     else
       []
     end
@@ -233,7 +253,6 @@ class Kamal::Configuration
     env_tags.detect { |t| t.name == name.to_s }
   end
 
-
   def to_h
     {
       roles: role_names,
@@ -248,8 +267,7 @@ class Kamal::Configuration
       sshkit: sshkit.to_h,
       builder: builder.to_h,
       accessories: raw_config.accessories,
-      logging: logging_args,
-      healthcheck: healthcheck.to_h
+      logging: logging_args
     }.compact
   end
 
@@ -264,22 +282,26 @@ class Kamal::Configuration
     end
 
     def ensure_required_keys_present
-      %i[ service image registry servers ].each do |key|
+      %i[ service image registry ].each do |key|
         raise Kamal::ConfigurationError, "Missing required configuration for #{key}" unless raw_config[key].present?
       end
 
-      unless role(primary_role_name).present?
-        raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
-      end
+      if raw_config.servers.nil?
+        raise Kamal::ConfigurationError, "No servers or accessories specified" unless raw_config.accessories.present?
+      else
+        unless role(primary_role_name).present?
+          raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
+        end
 
-      if primary_role.hosts.empty?
-        raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
-      end
+        if primary_role.hosts.empty?
+          raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
+        end
 
-      unless allow_empty_roles?
-        roles.each do |role|
-          if role.hosts.empty?
-            raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+        unless allow_empty_roles?
+          roles.each do |role|
+            if role.hosts.empty?
+              raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+            end
           end
         end
       end
@@ -307,6 +329,30 @@ class Kamal::Configuration
       true
     end
 
+    def ensure_no_traefik_reboot_hooks
+      hooks = %w[ pre-traefik-reboot post-traefik-reboot ].select { |hook_file| File.exist?(File.join(hooks_path, hook_file)) }
+
+      if hooks.any?
+        raise Kamal::ConfigurationError, "Found #{hooks.join(", ")}, these should be renamed to (pre|post)-proxy-reboot"
+      end
+
+      true
+    end
+
+    def ensure_one_host_for_ssl_roles
+      roles.each(&:ensure_one_host_for_ssl)
+
+      true
+    end
+
+    def ensure_unique_hosts_for_ssl_roles
+      hosts = roles.select(&:ssl?).flat_map { |role| role.proxy.hosts }
+      duplicates = hosts.tally.filter_map { |host, count| host if count > 1 }
+
+      raise Kamal::ConfigurationError, "Different roles can't share the same host for SSL: #{duplicates.join(", ")}" if duplicates.any?
+
+      true
+    end
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort

data/lib/kamal/docker.rb

@@ -0,0 +1,30 @@
+require "tempfile"
+require "open3"
+
+module Kamal::Docker
+  extend self
+  BUILD_CHECK_TAG = "kamal-local-build-check"
+
+  def included_files
+    Tempfile.create do |dockerfile|
+      dockerfile.write(<<~DOCKERFILE)
+        FROM busybox
+        COPY . app
+        WORKDIR app
+        CMD find . -type f | sed "s|^\./||"
+      DOCKERFILE
+      dockerfile.close
+
+      cmd = "docker buildx build -t=#{BUILD_CHECK_TAG} -f=#{dockerfile.path} ."
+      system(cmd) || raise("failed to build check image")
+    end
+
+    cmd = "docker run --rm #{BUILD_CHECK_TAG}"
+    out, err, status = Open3.capture3(cmd)
+    unless status
+      raise "failed to run check image:\n#{err}"
+    end
+
+    out.lines.map(&:strip)
+  end
+end

data/lib/kamal/env_file.rb

@@ -15,6 +15,10 @@ class Kamal::EnvFile
     env_file.presence || "\n"
   end
 
+  def to_io
+    StringIO.new(to_s)
+  end
+
   alias to_str to_s
 
   private
@@ -33,6 +37,8 @@ class Kamal::EnvFile
     def escape_docker_env_file_ascii_value(value)
       # Doublequotes are treated literally in docker env files
       # so remove leading and trailing ones and unescape any others
-      value.to_s.dump[1..-2].gsub(/\\"/, "\"")
+      value.to_s.dump[1..-2]
+        .gsub(/\\"/, "\"")
+        .gsub(/\\#/, "#")
     end
 end

data/lib/kamal/git.rb

@@ -24,4 +24,14 @@ module Kamal::Git
   def root
     `git rev-parse --show-toplevel`.strip
   end
+
+  # returns an array of relative path names of files with uncommitted changes
+  def uncommitted_files
+    `git ls-files --modified`.lines.map(&:strip)
+  end
+
+  # returns an array of relative path names of untracked files, including gitignored files
+  def untracked_files
+    `git ls-files --others`.lines.map(&:strip)
+  end
 end

data/lib/kamal/secrets/adapters/aws_secrets_manager.rb

@@ -0,0 +1,51 @@
+class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    def login(_account)
+      nil
+    end
+
+    def fetch_secrets(secrets, from:, account: nil, session:)
+      {}.tap do |results|
+        get_from_secrets_manager(prefixed_secrets(secrets, from: from), account: account).each do |secret|
+          secret_name = secret["Name"]
+          secret_string = JSON.parse(secret["SecretString"])
+
+          secret_string.each do |key, value|
+            results["#{secret_name}/#{key}"] = value
+          end
+        rescue JSON::ParserError
+          results["#{secret_name}"] = secret["SecretString"]
+        end
+      end
+    end
+
+    def get_from_secrets_manager(secrets, account: nil)
+      args = [ "aws", "secretsmanager", "batch-get-secret-value", "--secret-id-list" ] + secrets.map(&:shellescape)
+      args += [ "--profile", account.shellescape ] if account
+      args += [ "--output", "json" ]
+      cmd = args.join(" ")
+
+      `#{cmd}`.tap do |secrets|
+        raise RuntimeError, "Could not read #{secrets} from AWS Secrets Manager" unless $?.success?
+
+        secrets = JSON.parse(secrets)
+
+        return secrets["SecretValues"] unless secrets["Errors"].present?
+
+        raise RuntimeError, secrets["Errors"].map { |error| "#{error['SecretId']}: #{error['Message']}" }.join(" ")
+      end
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "AWS CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `aws --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/base.rb

@@ -0,0 +1,33 @@
+class Kamal::Secrets::Adapters::Base
+  delegate :optionize, to: Kamal::Utils
+
+  def fetch(secrets, account: nil, from: nil)
+    raise RuntimeError, "Missing required option '--account'" if requires_account? && account.blank?
+
+    check_dependencies!
+
+    session = login(account)
+    fetch_secrets(secrets, from: from, account: account, session: session)
+  end
+
+  def requires_account?
+    true
+  end
+
+  private
+    def login(...)
+      raise NotImplementedError
+    end
+
+    def fetch_secrets(...)
+      raise NotImplementedError
+    end
+
+    def check_dependencies!
+      raise NotImplementedError
+    end
+
+    def prefixed_secrets(secrets, from:)
+      secrets.map { |secret| [ from, secret ].compact.join("/") }
+    end
+end

data/lib/kamal/secrets/adapters/bitwarden.rb

@@ -0,0 +1,81 @@
+class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base
+  private
+    def login(account)
+      status = run_command("status")
+
+      if status["status"] == "unauthenticated"
+        run_command("login #{account.shellescape}", raw: true)
+        status = run_command("status")
+      end
+
+      if status["status"] == "locked"
+        session = run_command("unlock --raw", raw: true).presence
+        status = run_command("status", session: session)
+      end
+
+      raise RuntimeError, "Failed to login to and unlock Bitwarden" unless status["status"] == "unlocked"
+
+      run_command("sync", session: session, raw: true)
+      raise RuntimeError, "Failed to sync Bitwarden" unless $?.success?
+
+      session
+    end
+
+    def fetch_secrets(secrets, from:, account:, session:)
+      {}.tap do |results|
+        items_fields(prefixed_secrets(secrets, from: from)).each do |item, fields|
+          item_json = run_command("get item #{item.shellescape}", session: session, raw: true)
+          raise RuntimeError, "Could not read #{item} from Bitwarden" unless $?.success?
+          item_json = JSON.parse(item_json)
+          if fields.any?
+            results.merge! fetch_secrets_from_fields(fields, item, item_json)
+          elsif item_json.dig("login", "password")
+            results[item] = item_json.dig("login", "password")
+          elsif item_json["fields"]&.any?
+            fields = item_json["fields"].pluck("name")
+            results.merge! fetch_secrets_from_fields(fields, item, item_json)
+          else
+            raise RuntimeError, "Item #{item} is not a login type item and no fields were specified"
+          end
+        end
+      end
+    end
+
+    def fetch_secrets_from_fields(fields, item, item_json)
+      fields.to_h do |field|
+        item_field = item_json["fields"].find { |f| f["name"] == field }
+        raise RuntimeError, "Could not find field #{field} in item #{item} in Bitwarden" unless item_field
+        value = item_field["value"]
+        [ "#{item}/#{field}", value ]
+      end
+    end
+
+    def items_fields(secrets)
+      {}.tap do |items|
+        secrets.each do |secret|
+          item, field = secret.split("/")
+          items[item] ||= []
+          items[item] << field
+        end
+      end
+    end
+
+    def signedin?(account)
+      run_command("status")["status"] != "unauthenticated"
+    end
+
+    def run_command(command, session: nil, raw: false)
+      full_command = [ *("BW_SESSION=#{session.shellescape}" if session), "bw", command ].join(" ")
+      result = `#{full_command}`.strip
+      raw ? result : JSON.parse(result)
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Bitwarden CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `bw --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb

@@ -0,0 +1,66 @@
+class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    LIST_ALL_SELECTOR = "all"
+    LIST_ALL_FROM_PROJECT_SUFFIX = "/all"
+    LIST_COMMAND = "secret list"
+    GET_COMMAND = "secret get"
+
+    def fetch_secrets(secrets, from:, account:, session:)
+      raise RuntimeError, "You must specify what to retrieve from Bitwarden Secrets Manager" if secrets.length == 0
+
+      secrets = prefixed_secrets(secrets, from: from)
+      command, project = extract_command_and_project(secrets)
+
+      {}.tap do |results|
+        if command.nil?
+          secrets.each do |secret_uuid|
+            item_json = run_command("#{GET_COMMAND} #{secret_uuid.shellescape}")
+            raise RuntimeError, "Could not read #{secret_uuid} from Bitwarden Secrets Manager" unless $?.success?
+            item_json = JSON.parse(item_json)
+            results[item_json["key"]] = item_json["value"]
+          end
+        else
+          items_json = run_command(command)
+          raise RuntimeError, "Could not read secrets from Bitwarden Secrets Manager" unless $?.success?
+
+          JSON.parse(items_json).each do |item_json|
+            results[item_json["key"]] = item_json["value"]
+          end
+        end
+      end
+    end
+
+    def extract_command_and_project(secrets)
+      if secrets.length == 1
+        if secrets[0] == LIST_ALL_SELECTOR
+          [ LIST_COMMAND, nil ]
+        elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX)
+          project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first
+          [ "#{LIST_COMMAND} #{project.shellescape}", project ]
+        end
+      end
+    end
+
+    def run_command(command, session: nil)
+      full_command = [ "bws", command ].join(" ")
+      `#{full_command}`
+    end
+
+    def login(account)
+      run_command("project list")
+      raise RuntimeError, "Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?" unless $?.success?
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Bitwarden Secrets Manager CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `bws --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/doppler.rb

@@ -0,0 +1,57 @@
+class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    def login(*)
+      unless loggedin?
+        `doppler login -y`
+        raise RuntimeError, "Failed to login to Doppler" unless $?.success?
+      end
+    end
+
+    def loggedin?
+      `doppler me --json 2> /dev/null`
+      $?.success?
+    end
+
+    def fetch_secrets(secrets, from:, **)
+      secrets = prefixed_secrets(secrets, from: from)
+      flags = secrets_get_flags(secrets)
+
+      secret_names = secrets.collect { |s| s.split("/").last }
+
+      items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{flags}`
+      raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success?
+
+      items = JSON.parse(items)
+
+      items.transform_values { |value| value["computed"] }
+    end
+
+    def secrets_get_flags(secrets)
+      unless service_token_set?
+        project, config, _ = secrets.first.split("/")
+
+        unless project && config
+          raise RuntimeError, "Missing project or config from '--from=project/config' option"
+        end
+
+        project_and_config_flags = "-p #{project.shellescape} -c #{config.shellescape}"
+      end
+    end
+
+    def service_token_set?
+      ENV["DOPPLER_TOKEN"] && ENV["DOPPLER_TOKEN"][0, 5] == "dp.st"
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Doppler CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `doppler --version 2> /dev/null`
+      $?.success?
+    end
+end