kamal 1.8.3 → 2.7.0

data/lib/kamal/configuration/docs/registry.yml

@@ -1,10 +1,13 @@
 # Registry
 #
-# The default registry is Docker Hub, but you can change it using registry/server:
+# The default registry is Docker Hub, but you can change it using `registry/server`.
 #
-# A reference to secret (in this case DOCKER_REGISTRY_TOKEN) will look up the secret
-# in the local environment.
-
+# By default, Docker Hub creates public repositories. To avoid making your images public,
+# set up a private repository before deploying, or change the default repository privacy
+# settings to private in your [Docker Hub settings](https://hub.docker.com/repository-settings/default-privacy).
+#
+# A reference to a secret (in this case, `DOCKER_REGISTRY_TOKEN`) will look up the secret
+# in the local environment:
 registry:
   server: registry.digitalocean.com
   username:
@@ -13,28 +16,31 @@ registry:
     - DOCKER_REGISTRY_TOKEN
 
 # Using AWS ECR as the container registry
-# You will need to have the aws CLI installed locally for this to work.
-# AWS ECR’s access token is only valid for 12hrs. In order to not have to manually regenerate the token every time, you can use ERB in the deploy.yml file to shell out to the aws cli command, and obtain the token:
-
+#
+# You will need to have the AWS CLI installed locally for this to work.
+# AWS ECR’s access token is only valid for 12 hours. In order to avoid having to manually regenerate the token every time, you can use ERB in the `deploy.yml` file to shell out to the AWS CLI command and obtain the token:
 registry:
   server: <your aws account id>.dkr.ecr.<your aws region id>.amazonaws.com
   username: AWS
   password: <%= %x(aws ecr get-login-password) %>
 
 # Using GCP Artifact Registry as the container registry
-# To sign into Artifact Registry, you would need to
+#
+# To sign into Artifact Registry, you need to
 # [create a service account](https://cloud.google.com/iam/docs/service-accounts-create#creating)
 # and [set up roles and permissions](https://cloud.google.com/artifact-registry/docs/access-control#permissions).
-# Normally, assigning a roles/artifactregistry.writer role should be sufficient.
+# Normally, assigning the `roles/artifactregistry.writer` role should be sufficient.
 #
-# Once the service account is ready, you need to generate and download a JSON key, base64 encode it and add to .env:
+# Once the service account is ready, you need to generate and download a JSON key and base64 encode it:
 #
 # ```shell
-# echo "KAMAL_REGISTRY_PASSWORD=$(base64 -i /path/to/key.json)" | tr -d "\\n"  >> .env
+# base64 -i /path/to/key.json | tr -d "\\n"
 # ```
-# Use the env variable as password along with _json_key_base64 as username.
+#
+# You'll then need to set the `KAMAL_REGISTRY_PASSWORD` secret to that value.
+#
+# Use the environment variable as the password along with `_json_key_base64` as the username.
 # Here’s the final configuration:
-
 registry:
   server: <your registry region>-docker.pkg.dev
   username: _json_key_base64
@@ -44,6 +50,7 @@ registry:
 # Validating the configuration
 #
 # You can validate the configuration by running:
+#
 # ```shell
 # kamal registry login
 # ```

data/lib/kamal/configuration/docs/role.yml

@@ -1,22 +1,21 @@
 # Roles
 #
 # Roles are used to configure different types of servers in the deployment.
-# The most common use for this is to run a web servers and job servers.
+# The most common use for this is to run web servers and job servers.
 #
 # Kamal expects there to be a `web` role, unless you set a different `primary_role`
 # in the root configuration.
 
 # Role configuration
 #
-# Roles are specified under the servers key
+# Roles are specified under the servers key:
 servers:
 
   # Simple role configuration
   #
+  # This can be a list of hosts if you don't need custom configuration for the role.
   #
-  # This can be a list of hosts, if you don't need custom configuration for the role.
-  #
-  # You can set tags on the hosts for custom env variables (see kamal docs env)
+  # You can set tags on the hosts for custom env variables (see kamal docs env):
   web:
     - 172.1.0.1
     - 172.1.0.2: experiment1
@@ -24,29 +23,31 @@ servers:
 
   # Custom role configuration
   #
-  # When there are other options to set, the list of hosts goes under the `hosts` key
+  # When there are other options to set, the list of hosts goes under the `hosts` key.
+  #
+  # By default, only the primary role uses a proxy.
+  #
+  # For other roles, you can set it to `proxy: true` to enable it and inherit the root proxy
+  # configuration or provide a map of options to override the root configuration.
   #
-  # By default only the primary role uses Traefik, but you can set `traefik` to change
-  # it.
+  # For the primary role, you can set `proxy: false` to disable the proxy.
   #
-  # You can also set a custom cmd to run in the container, and overwrite other settings
+  # You can also set a custom `cmd` to run in the container and overwrite other settings
   # from the root configuration.
   workers:
     hosts:
       - 172.1.0.3
       - 172.1.0.4: experiment1
-    traefik: true
     cmd: "bin/jobs"
     options:
       memory: 2g
       cpus: 4
-    healthcheck:
-      ...
     logging:
       ...
+    proxy:
+      ...
     labels:
       my-label: workers
     env:
       ...
     asset_path: /public
-

data/lib/kamal/configuration/docs/servers.yml

@@ -2,7 +2,7 @@
 #
 # Servers are split into different roles, with each role having its own configuration.
 #
-# For simpler deployments though where all servers are identical, you can just specify a list of servers
+# For simpler deployments, though, where all servers are identical, you can just specify a list of servers.
 # They will be implicitly assigned to the `web` role.
 servers:
   - 172.0.0.1
@@ -19,7 +19,7 @@ servers:
 
 # Roles
 #
-# For more complex deployments (e.g. if you are running job hosts), you can specify roles, and configure each separately (see kamal docs role)
+# For more complex deployments (e.g., if you are running job hosts), you can specify roles and configure each separately (see kamal docs role):
 servers:
   web:
     ...

data/lib/kamal/configuration/docs/ssh.yml

@@ -1,9 +1,9 @@
 # SSH configuration
 #
-# Kamal uses SSH to connect run commands on your hosts.
-# By default it will attempt to connect to the root user on port 22
+# Kamal uses SSH to connect and run commands on your hosts.
+# By default, it will attempt to connect to the root user on port 22.
 #
-# If you are using non-root user, you may need to bootstrap your servers manually, before using them with Kamal. On Ubuntu, you’d do:
+# If you are using a non-root user, you may need to bootstrap your servers manually before using them with Kamal. On Ubuntu, you’d do:
 #
 # ```shell
 # sudo apt update
@@ -12,7 +12,6 @@
 # sudo usermod -a -G docker app
 # ```
 
-
 # SSH options
 #
 # The options are specified under the ssh key in the configuration file.
@@ -20,47 +19,52 @@ ssh:
 
   # The SSH user
   #
-  # Defaults to `root`
-  #
+  # Defaults to `root`:
   user: app
 
   # The SSH port
   #
-  # Defaults to 22
+  # Defaults to 22:
   port: "2222"
 
   # Proxy host
   #
-  # Specified in the form <host> or <user>@<host>
+  # Specified in the form <host> or <user>@<host>:
   proxy: root@proxy-host
 
   # Proxy command
   #
-  # A custom proxy command, required for older versions of SSH
+  # A custom proxy command, required for older versions of SSH:
   proxy_command: "ssh -W %h:%p user@proxy"
 
   # Log level
   #
-  # Defaults to `fatal`. Set this to debug if you are having
-  # SSH connection issues.
+  # Defaults to `fatal`. Set this to `debug` if you are having SSH connection issues.
   log_level: debug
 
-  # Keys Only
+  # Keys only
   #
-  # Set to true to use only private keys from keys and key_data parameters, 
-  # even if ssh-agent offers more identities. This option is intended for 
-  # situations where ssh-agent offers many different identites or you have 
-  # a need to overwrite all identites and force a single one.
+  # Set to `true` to use only private keys from the `keys` and `key_data` parameters,
+  # even if ssh-agent offers more identities. This option is intended for
+  # situations where ssh-agent offers many different identities or you
+  # need to overwrite all identities and force a single one.
   keys_only: false
 
   # Keys
   #
-  # An array of file names of private keys to use for publickey
-  # and hostbased authentication
+  # An array of file names of private keys to use for public key
+  # and host-based authentication:
   keys: [ "~/.ssh/id.pem" ]
 
-  # Key Data
+  # Key data
   #
   # An array of strings, with each element of the array being
   # a raw private key in PEM format.
   key_data: [ "-----BEGIN OPENSSH PRIVATE KEY-----" ]
+
+  # Config
+  #
+  # Set to true to load the default OpenSSH config files (~/.ssh/config,
+  # /etc/ssh_config), to false ignore config files, or to a file path
+  # (or array of paths) to load specific configuration. Defaults to true.
+  config: true

data/lib/kamal/configuration/docs/sshkit.yml

@@ -2,8 +2,8 @@
 #
 # [SSHKit](https://github.com/capistrano/sshkit) is the SSH toolkit used by Kamal.
 #
-# The default settings should be sufficient for most use cases, but
-# when connecting to a large number of hosts you may need to adjust
+# The default, settings should be sufficient for most use cases, but
+# when connecting to a large number of hosts, you may need to adjust.
 
 # SSHKit options
 #
@@ -13,11 +13,11 @@ sshkit:
   # Max concurrent starts
   #
   # Creating SSH connections concurrently can be an issue when deploying to many servers.
-  # By default Kamal will limit concurrent connection starts to 30 at a time.
+  # By default, Kamal will limit concurrent connection starts to 30 at a time.
   max_concurrent_starts: 10
 
   # Pool idle timeout
   #
   # Kamal sets a long idle timeout of 900 seconds on connections to try to avoid
-  # re-connection storms after an idle period, like building an image or waiting for CI.
+  # re-connection storms after an idle period, such as building an image or waiting for CI.
   pool_idle_timeout: 300

data/lib/kamal/configuration/env/tag.rb

@@ -1,12 +1,13 @@
 class Kamal::Configuration::Env::Tag
-  attr_reader :name, :config
+  attr_reader :name, :config, :secrets
 
-  def initialize(name, config:)
+  def initialize(name, config:, secrets:)
     @name = name
     @config = config
+    @secrets = secrets
   end
 
   def env
-    Kamal::Configuration::Env.new(config: config)
+    Kamal::Configuration::Env.new(config: config, secrets: secrets)
   end
 end

data/lib/kamal/configuration/env.rb

@@ -1,36 +1,38 @@
 class Kamal::Configuration::Env
   include Kamal::Configuration::Validation
 
-  attr_reader :secrets_keys, :clear, :secrets_file, :context
+  attr_reader :context, :clear, :secret_keys
   delegate :argumentize, to: Kamal::Utils
 
-  def initialize(config:, secrets_file: nil, context: "env")
+  def initialize(config:, secrets:, context: "env")
     @clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)
-    @secrets_keys = config.fetch("secret", [])
-    @secrets_file = secrets_file
+    @secrets = secrets
+    @secret_keys = config.fetch("secret", [])
     @context = context
     validate! config, context: context, with: Kamal::Configuration::Validator::Env
   end
 
-  def args
-    [ "--env-file", secrets_file, *argumentize("--env", clear) ]
+  def clear_args
+    argumentize("--env", clear)
   end
 
   def secrets_io
-    StringIO.new(Kamal::EnvFile.new(secrets).to_s)
-  end
-
-  def secrets
-    @secrets ||= secrets_keys.to_h { |key| [ key, ENV.fetch(key) ] }
-  end
-
-  def secrets_directory
-    File.dirname(secrets_file)
+    Kamal::EnvFile.new(aliased_secrets).to_io
   end
 
   def merge(other)
     self.class.new \
-      config: { "clear" => clear.merge(other.clear), "secret" => secrets_keys | other.secrets_keys },
-      secrets_file: secrets_file || other.secrets_file
+      config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
+      secrets: @secrets
   end
+
+  private
+    def aliased_secrets
+      secret_keys.to_h { |key| extract_alias(key) }.transform_values { |secret_key| @secrets[secret_key] }
+    end
+
+    def extract_alias(key)
+      key_name, key_aliased_to = key.split(":", 2)
+      [ key_name, key_aliased_to || key_name ]
+    end
 end

data/lib/kamal/configuration/proxy/boot.rb

@@ -0,0 +1,129 @@
+class Kamal::Configuration::Proxy::Boot
+  MINIMUM_VERSION = "v0.9.0"
+  DEFAULT_HTTP_PORT = 80
+  DEFAULT_HTTPS_PORT = 443
+  DEFAULT_LOG_MAX_SIZE = "10m"
+
+  attr_reader :config
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  def initialize(config:)
+    @config = config
+  end
+
+  def publish_args(http_port, https_port, bind_ips = nil)
+    ensure_valid_bind_ips(bind_ips)
+
+    (bind_ips || [ nil ]).map do |bind_ip|
+      bind_ip = format_bind_ip(bind_ip)
+      publish_http = [ bind_ip, http_port, DEFAULT_HTTP_PORT ].compact.join(":")
+      publish_https = [ bind_ip, https_port, DEFAULT_HTTPS_PORT ].compact.join(":")
+
+      argumentize "--publish", [ publish_http, publish_https ]
+    end.join(" ")
+  end
+
+  def logging_args(max_size)
+    argumentize "--log-opt", "max-size=#{max_size}" if max_size.present?
+  end
+
+  def default_boot_options
+    [
+      *(publish_args(DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, nil)),
+      *(logging_args(DEFAULT_LOG_MAX_SIZE))
+    ]
+  end
+
+  def repository_name
+    "basecamp"
+  end
+
+  def image_name
+    "kamal-proxy"
+  end
+
+  def image_default
+    "#{repository_name}/#{image_name}"
+  end
+
+  def container_name
+    "kamal-proxy"
+  end
+
+  def host_directory
+    File.join config.run_directory, "proxy"
+  end
+
+  def options_file
+    File.join host_directory, "options"
+  end
+
+  def image_file
+    File.join host_directory, "image"
+  end
+
+  def image_version_file
+    File.join host_directory, "image_version"
+  end
+
+  def run_command_file
+    File.join host_directory, "run_command"
+  end
+
+  def apps_directory
+    File.join host_directory, "apps-config"
+  end
+
+  def apps_container_directory
+    "/home/kamal-proxy/.apps-config"
+  end
+
+  def apps_volume
+    Kamal::Configuration::Volume.new \
+      host_path: apps_directory,
+      container_path: apps_container_directory
+  end
+
+  def app_directory
+    File.join apps_directory, config.service_and_destination
+  end
+
+  def app_container_directory
+    File.join apps_container_directory, config.service_and_destination
+  end
+
+  def error_pages_directory
+    File.join app_directory, "error_pages"
+  end
+
+  def error_pages_container_directory
+    File.join app_container_directory, "error_pages"
+  end
+
+  def tls_directory
+    File.join app_directory, "tls"
+  end
+
+  def tls_container_directory
+    File.join app_container_directory, "tls"
+  end
+
+  private
+    def ensure_valid_bind_ips(bind_ips)
+      bind_ips.present? && bind_ips.each do |ip|
+        next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex
+        raise ArgumentError, "Invalid publish IP address: #{ip}"
+      end
+
+      true
+    end
+
+    def format_bind_ip(ip)
+      # Ensure IPv6 address inside square brackets - e.g. [::1]
+      if ip =~ Resolv::IPv6::Regex && ip !~ /\A\[.*\]\z/
+        "[#{ip}]"
+      else
+        ip
+      end
+    end
+end

data/lib/kamal/configuration/proxy.rb

@@ -0,0 +1,124 @@
+class Kamal::Configuration::Proxy
+  include Kamal::Configuration::Validation
+
+  DEFAULT_LOG_REQUEST_HEADERS = [ "Cache-Control", "Last-Modified", "User-Agent" ]
+  CONTAINER_NAME = "kamal-proxy"
+
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  attr_reader :config, :proxy_config, :role_name, :secrets
+
+  def initialize(config:, proxy_config:, role_name: nil, secrets:, context: "proxy")
+    @config = config
+    @proxy_config = proxy_config
+    @proxy_config = {} if @proxy_config.nil?
+    @role_name = role_name
+    @secrets = secrets
+    validate! @proxy_config, with: Kamal::Configuration::Validator::Proxy, context: context
+  end
+
+  def app_port
+    proxy_config.fetch("app_port", 80)
+  end
+
+  def ssl?
+    proxy_config.fetch("ssl", false)
+  end
+
+  def hosts
+    proxy_config["hosts"] || proxy_config["host"]&.split(",") || []
+  end
+
+  def custom_ssl_certificate?
+    ssl = proxy_config["ssl"]
+    return false unless ssl.is_a?(Hash)
+    ssl["certificate_pem"].present? && ssl["private_key_pem"].present?
+  end
+
+  def certificate_pem_content
+    ssl = proxy_config["ssl"]
+    return nil unless ssl.is_a?(Hash)
+    secrets[ssl["certificate_pem"]]
+  end
+
+  def private_key_pem_content
+    ssl = proxy_config["ssl"]
+    return nil unless ssl.is_a?(Hash)
+    secrets[ssl["private_key_pem"]]
+  end
+
+  def host_tls_cert
+    tls_path(config.proxy_boot.tls_directory, "cert.pem")
+  end
+
+  def host_tls_key
+    tls_path(config.proxy_boot.tls_directory, "key.pem")
+  end
+
+  def container_tls_cert
+    tls_path(config.proxy_boot.tls_container_directory, "cert.pem")
+  end
+
+  def container_tls_key
+    tls_path(config.proxy_boot.tls_container_directory, "key.pem") if custom_ssl_certificate?
+  end
+
+  def deploy_options
+    {
+      host: hosts,
+      tls: ssl? ? true : nil,
+      "tls-certificate-path": container_tls_cert,
+      "tls-private-key-path": container_tls_key,
+      "deploy-timeout": seconds_duration(config.deploy_timeout),
+      "drain-timeout": seconds_duration(config.drain_timeout),
+      "health-check-interval": seconds_duration(proxy_config.dig("healthcheck", "interval")),
+      "health-check-timeout": seconds_duration(proxy_config.dig("healthcheck", "timeout")),
+      "health-check-path": proxy_config.dig("healthcheck", "path"),
+      "target-timeout": seconds_duration(proxy_config["response_timeout"]),
+      "buffer-requests": proxy_config.fetch("buffering", { "requests": true }).fetch("requests", true),
+      "buffer-responses": proxy_config.fetch("buffering", { "responses": true }).fetch("responses", true),
+      "buffer-memory": proxy_config.dig("buffering", "memory"),
+      "max-request-body": proxy_config.dig("buffering", "max_request_body"),
+      "max-response-body": proxy_config.dig("buffering", "max_response_body"),
+      "path-prefix": proxy_config.dig("path_prefix"),
+      "strip-path-prefix": proxy_config.dig("strip_path_prefix"),
+      "forward-headers": proxy_config.dig("forward_headers"),
+      "tls-redirect": proxy_config.dig("ssl_redirect"),
+      "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS,
+      "log-response-header": proxy_config.dig("logging", "response_headers"),
+      "error-pages": error_pages
+    }.compact
+  end
+
+  def deploy_command_args(target:)
+    optionize ({ target: "#{target}:#{app_port}" }).merge(deploy_options), with: "="
+  end
+
+  def stop_options(drain_timeout: nil, message: nil)
+    {
+      "drain-timeout": seconds_duration(drain_timeout),
+      message: message
+    }.compact
+  end
+
+  def stop_command_args(**options)
+    optionize stop_options(**options), with: "="
+  end
+
+  def merge(other)
+    self.class.new config: config, proxy_config: other.proxy_config.deep_merge(proxy_config), role_name: role_name, secrets: secrets
+  end
+
+  private
+    def tls_path(directory, filename)
+      File.join([ directory, role_name, filename ].compact) if custom_ssl_certificate?
+    end
+
+    def seconds_duration(value)
+      value ? "#{value}s" : nil
+    end
+
+    def error_pages
+      File.join config.proxy_boot.error_pages_container_directory, config.version if config.error_pages_path
+    end
+end

data/lib/kamal/configuration/registry.rb

@@ -1,11 +1,10 @@
 class Kamal::Configuration::Registry
   include Kamal::Configuration::Validation
 
-  attr_reader :registry_config
-
-  def initialize(config:)
-    @registry_config = config.raw_config.registry || {}
-    validate! registry_config, with: Kamal::Configuration::Validator::Registry
+  def initialize(config:, secrets:, context: "registry")
+    @registry_config = config["registry"] || {}
+    @secrets = secrets
+    validate! registry_config, context: context, with: Kamal::Configuration::Validator::Registry
   end
 
   def server
@@ -21,9 +20,11 @@ class Kamal::Configuration::Registry
   end
 
   private
+    attr_reader :registry_config, :secrets
+
     def lookup(key)
       if registry_config[key].is_a?(Array)
-        ENV.fetch(registry_config[key].first).dup
+        secrets[registry_config[key].first]
       else
         registry_config[key]
       end