kamal 2.0.0.alpha → 2.0.0.beta2

data/lib/kamal/configuration/docs/proxy.yml

@@ -0,0 +1,100 @@
+# Proxy
+#
+# Kamal uses [kamal-proxy](https://github.com/basecamp/kamal-proxy) to provide
+# gapless deployments. It runs on ports 80 and 443 and forwards requests to the
+# application container.
+#
+# The proxy is configured in the root configuration under `proxy`. These are
+# options that are set when deploying the application, not when booting the proxy
+#
+# They are application specific, so are not shared when multiple applications
+# run on the same proxy.
+#
+# The proxy is enabled by default on the primary role, but can be disabled by
+# setting `proxy: false`.
+#
+# It is disabled by default on all other roles, but can be enabled by setting
+# `proxy: true`, or providing a proxy configuration.
+proxy:
+
+  # Host
+  #
+  # The hosts that will be used to serve the app. The proxy will only route requests
+  # to this host to your app.
+  #
+  # If no hosts are set, then all requests will be forwarded, except for matching
+  # requests for other apps deployed on that server that do have a host set.
+  host: foo.example.com
+
+  # App port
+  #
+  # The port the application container is exposed on
+  #
+  # Defaults to 80
+  app_port: 3000
+
+  # SSL
+  #
+  # kamal-proxy can provide automatic HTTPS for your application via Let's Encrypt.
+  #
+  # This requires that we are deploying to a one server and the host option is set.
+  # The host value must point to the server we are deploying to and port 443 must be
+  # open for the Let's Encrypt challenge to succeed.
+  #
+  # Defaults to false
+  ssl: true
+
+  # Response timeout
+  #
+  # How long to wait for requests to complete before timing out, defaults to 30 seconds
+  response_timeout: 10s
+
+  # Healthcheck
+  #
+  # When deploying, the proxy will by default hit /up once every second until we hit
+  # the deploy timeout, with a 5 second timeout for each request.
+  #
+  # Once the app is up, the proxy will stop hitting the healthcheck endpoint.
+  healthcheck:
+    interval: 3
+    path: /health
+    timeout: 3
+
+  # Buffering
+  #
+  # Whether to buffer request and response bodies in the proxy
+  #
+  # By default buffering is enabled with a max request body size of 1GB and no limit
+  # for response size.
+  #
+  # You can also set the memory limit for buffering, which defaults to 1MB, anything
+  # larger than that is written to disk.
+  buffering:
+    requests: true
+    responses: true
+    max_request_body: 40_000_000
+    max_response_body: 0
+    memory: 2_000_000
+
+  # Logging
+  #
+  # Configure request logging for the proxy
+  # You can specify request and response headers to log.
+  # By default, Cache-Control, Last-Modified and User-Agent request headers are logged
+  logging:
+    request_headers:
+      - Cache-Control
+      - X-Forwarded-Proto
+    response_headers:
+      - X-Request-ID
+      - X-Request-Start
+
+  # Forward headers
+  #
+  # Whether to forward the X-Forwarded-For and X-Forwarded-Proto headers (defaults to false)
+  #
+  # If you are behind a trusted proxy, you can set this to true to forward the headers.
+  #
+  # By default kamal-proxy will not forward the headers the ssl option is set to true, and
+  # will forward them if it is set to false.
+  forward_headers: true

data/lib/kamal/configuration/docs/registry.yml

@@ -27,11 +27,13 @@ registry:
 # and [set up roles and permissions](https://cloud.google.com/artifact-registry/docs/access-control#permissions).
 # Normally, assigning a roles/artifactregistry.writer role should be sufficient.
 #
-# Once the service account is ready, you need to generate and download a JSON key, base64 encode it and add to .env:
+# Once the service account is ready, you need to generate and download a JSON key and base64 encode it:
 #
 # ```shell
-# echo "KAMAL_REGISTRY_PASSWORD=$(base64 -i /path/to/key.json)" | tr -d "\\n"  >> .env
+# base64 -i /path/to/key.json | tr -d "\\n")
 # ```
+# You'll then need to set the KAMAL_REGISTRY_PASSWORD secret to that value.
+#
 # Use the env variable as password along with _json_key_base64 as username.
 # Here’s the final configuration:
 

data/lib/kamal/configuration/docs/role.yml

@@ -26,7 +26,7 @@ servers:
   #
   # When there are other options to set, the list of hosts goes under the `hosts` key
   #
-  # By default only the primary role uses Traefik, but you can set `traefik` to change
+  # By default only the primary role uses a proxy, but you can set `proxy` to change
   # it.
   #
   # You can also set a custom cmd to run in the container, and overwrite other settings
@@ -35,18 +35,16 @@ servers:
     hosts:
       - 172.1.0.3
       - 172.1.0.4: experiment1
-    traefik: true
     cmd: "bin/jobs"
     options:
       memory: 2g
       cpus: 4
-    healthcheck:
-      ...
     logging:
       ...
+    proxy:
+      ...
     labels:
       my-label: workers
     env:
       ...
     asset_path: /public
-

data/lib/kamal/configuration/env/tag.rb

@@ -1,12 +1,13 @@
 class Kamal::Configuration::Env::Tag
-  attr_reader :name, :config
+  attr_reader :name, :config, :secrets
 
-  def initialize(name, config:)
+  def initialize(name, config:, secrets:)
     @name = name
     @config = config
+    @secrets = secrets
   end
 
   def env
-    Kamal::Configuration::Env.new(config: config)
+    Kamal::Configuration::Env.new(config: config, secrets: secrets)
   end
 end

data/lib/kamal/configuration/env.rb

@@ -1,36 +1,29 @@
 class Kamal::Configuration::Env
   include Kamal::Configuration::Validation
 
-  attr_reader :secrets_keys, :clear, :secrets_file, :context
+  attr_reader :context, :secrets
+  attr_reader :clear, :secret_keys
   delegate :argumentize, to: Kamal::Utils
 
-  def initialize(config:, secrets_file: nil, context: "env")
+  def initialize(config:, secrets:, context: "env")
     @clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)
-    @secrets_keys = config.fetch("secret", [])
-    @secrets_file = secrets_file
+    @secrets = secrets
+    @secret_keys = config.fetch("secret", [])
     @context = context
     validate! config, context: context, with: Kamal::Configuration::Validator::Env
   end
 
-  def args
-    [ "--env-file", secrets_file, *argumentize("--env", clear) ]
+  def clear_args
+    argumentize("--env", clear)
   end
 
   def secrets_io
-    StringIO.new(Kamal::EnvFile.new(secrets).to_s)
-  end
-
-  def secrets
-    @secrets ||= secrets_keys.to_h { |key| [ key, ENV.fetch(key) ] }
-  end
-
-  def secrets_directory
-    File.dirname(secrets_file)
+    Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io
   end
 
   def merge(other)
     self.class.new \
-      config: { "clear" => clear.merge(other.clear), "secret" => secrets_keys | other.secrets_keys },
-      secrets_file: secrets_file || other.secrets_file
+      config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
+      secrets: secrets
   end
 end

data/lib/kamal/configuration/proxy.rb

@@ -0,0 +1,66 @@
+class Kamal::Configuration::Proxy
+  include Kamal::Configuration::Validation
+
+  DEFAULT_LOG_REQUEST_HEADERS = [ "Cache-Control", "Last-Modified", "User-Agent" ]
+  CONTAINER_NAME = "kamal-proxy"
+
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  attr_reader :config, :proxy_config
+
+  def initialize(config:, proxy_config:, context: "proxy")
+    @config = config
+    @proxy_config = proxy_config
+    validate! @proxy_config, with: Kamal::Configuration::Validator::Proxy, context: context
+  end
+
+  def app_port
+    proxy_config.fetch("app_port", 80)
+  end
+
+  def ssl?
+    proxy_config.fetch("ssl", false)
+  end
+
+  def host
+    proxy_config["host"]
+  end
+
+  def deploy_options
+    {
+      host: proxy_config["host"],
+      tls: proxy_config["ssl"],
+      "deploy-timeout": seconds_duration(config.deploy_timeout),
+      "drain-timeout": seconds_duration(config.drain_timeout),
+      "health-check-interval": seconds_duration(proxy_config.dig("healthcheck", "interval")),
+      "health-check-timeout": seconds_duration(proxy_config.dig("healthcheck", "timeout")),
+      "health-check-path": proxy_config.dig("healthcheck", "path"),
+      "target-timeout": seconds_duration(proxy_config["response_timeout"]),
+      "buffer-requests": proxy_config.fetch("buffering", { "requests": true }).fetch("requests", true),
+      "buffer-responses": proxy_config.fetch("buffering", { "responses": true }).fetch("responses", true),
+      "buffer-memory": proxy_config.dig("buffering", "memory"),
+      "max-request-body": proxy_config.dig("buffering", "max_request_body"),
+      "max-response-body": proxy_config.dig("buffering", "max_response_body"),
+      "forward-headers": proxy_config.dig("forward_headers"),
+      "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS,
+      "log-response-header": proxy_config.dig("logging", "response_headers")
+    }.compact
+  end
+
+  def deploy_command_args(target:)
+    optionize ({ target: "#{target}:#{app_port}" }).merge(deploy_options)
+  end
+
+  def remove_command_args(target:)
+    optionize({ target: "#{target}:#{app_port}" })
+  end
+
+  def merge(other)
+    self.class.new config: config, proxy_config: proxy_config.deep_merge(other.proxy_config)
+  end
+
+  private
+    def seconds_duration(value)
+      value ? "#{value}s" : nil
+    end
+end

data/lib/kamal/configuration/registry.rb

@@ -1,10 +1,11 @@
 class Kamal::Configuration::Registry
   include Kamal::Configuration::Validation
 
-  attr_reader :registry_config
+  attr_reader :registry_config, :secrets
 
   def initialize(config:)
     @registry_config = config.raw_config.registry || {}
+    @secrets = config.secrets
     validate! registry_config, with: Kamal::Configuration::Validator::Registry
   end
 
@@ -23,7 +24,7 @@ class Kamal::Configuration::Registry
   private
     def lookup(key)
       if registry_config[key].is_a?(Array)
-        ENV.fetch(registry_config[key].first).dup
+        secrets[registry_config[key].first]
       else
         registry_config[key]
       end

data/lib/kamal/configuration/role.rb

@@ -1,10 +1,9 @@
 class Kamal::Configuration::Role
   include Kamal::Configuration::Validation
 
-  CORD_FILE = "cord"
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_reader :name, :config, :specialized_env, :specialized_logging, :specialized_healthcheck
+  attr_reader :name, :config, :specialized_env, :specialized_logging, :specialized_proxy
 
   alias to_s name
 
@@ -18,16 +17,14 @@ class Kamal::Configuration::Role
 
     @specialized_env = Kamal::Configuration::Env.new \
       config: specializations.fetch("env", {}),
-      secrets_file: File.join(config.host_env_directory, "roles", "#{container_prefix}.env"),
+      secrets: config.secrets,
       context: "servers/#{name}/env"
 
     @specialized_logging = Kamal::Configuration::Logging.new \
       logging_config: specializations.fetch("logging", {}),
       context: "servers/#{name}/logging"
 
-    @specialized_healthcheck = Kamal::Configuration::Healthcheck.new \
-      healthcheck_config: specializations.fetch("healthcheck", {}),
-      context: "servers/#{name}/healthcheck"
+    initialize_specialized_proxy
   end
 
   def primary_host
@@ -55,7 +52,7 @@ class Kamal::Configuration::Role
   end
 
   def labels
-    default_labels.merge(traefik_labels).merge(custom_labels)
+    default_labels.merge(custom_labels)
   end
 
   def label_args
@@ -70,87 +67,53 @@ class Kamal::Configuration::Role
     @logging ||= config.logging.merge(specialized_logging)
   end
 
-
-  def env(host)
-    @envs ||= {}
-    @envs[host] ||= [ config.env, specialized_env, *env_tags(host).map(&:env) ].reduce(:merge)
+  def proxy
+    @proxy ||= config.proxy.merge(specialized_proxy) if running_proxy?
   end
 
-  def env_args(host)
-    env(host).args
+  def running_proxy?
+    @running_proxy
   end
 
-  def asset_volume_args
-    asset_volume&.docker_args
+  def ssl?
+    running_proxy? && proxy.ssl?
   end
 
+  def stop_args
+    # When deploying with the proxy, kamal-proxy will drain request before returning so we don't need to wait.
+    timeout = running_proxy? ? nil : config.drain_timeout
 
-  def health_check_args(cord: true)
-    if running_traefik? || healthcheck.set_port_or_path?
-      if cord && uses_cord?
-        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => healthcheck.interval })
-          .concat(cord_volume.docker_args)
-      else
-        optionize({ "health-cmd" => healthcheck.cmd, "health-interval" => healthcheck.interval })
-      end
-    else
-      []
-    end
+    [ *argumentize("-t", timeout) ]
   end
 
-  def healthcheck
-    @healthcheck ||=
-      if running_traefik?
-        config.healthcheck.merge(specialized_healthcheck)
-      else
-        specialized_healthcheck
-      end
-  end
-
-  def health_check_cmd_with_cord
-    "(#{healthcheck.cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
-  end
-
-
-  def running_traefik?
-    if specializations["traefik"].nil?
-      primary?
-    else
-      specializations["traefik"]
-    end
+  def env(host)
+    @envs ||= {}
+    @envs[host] ||= [ config.env, specialized_env, *env_tags(host).map(&:env) ].reduce(:merge)
   end
 
-  def primary?
-    self == @config.primary_role
+  def env_args(host)
+    [ *env(host).clear_args, *argumentize("--env-file", secrets_path) ]
   end
 
-
-  def uses_cord?
-    running_traefik? && cord_volume && healthcheck.cmd.present?
+  def env_directory
+    File.join(config.env_directory, "roles")
   end
 
-  def cord_host_directory
-    File.join config.run_directory_as_docker_volume, "cords", [ container_prefix, config.run_id ].join("-")
+  def secrets_io(host)
+    env(host).secrets_io
   end
 
-  def cord_volume
-    if (cord = healthcheck.cord)
-      @cord_volume ||= Kamal::Configuration::Volume.new \
-        host_path: File.join(config.run_directory, "cords", [ container_prefix, config.run_id ].join("-")),
-        container_path: cord
-    end
+  def secrets_path
+    File.join(config.env_directory, "roles", "#{name}.env")
   end
 
-  def cord_host_file
-    File.join cord_volume.host_path, CORD_FILE
+  def asset_volume_args
+    asset_volume&.docker_args
   end
 
-  def cord_container_directory
-    health_check_options.fetch("cord", nil)
-  end
 
-  def cord_container_file
-    File.join cord_volume.container_path, CORD_FILE
+  def primary?
+    name == @config.primary_role_name
   end
 
 
@@ -168,25 +131,52 @@ class Kamal::Configuration::Role
   end
 
   def assets?
-    asset_path.present? && running_traefik?
+    asset_path.present? && running_proxy?
   end
 
-  def asset_volume(version = nil)
+  def asset_volume(version = config.version)
     if assets?
       Kamal::Configuration::Volume.new \
-        host_path: asset_volume_path(version), container_path: asset_path
+        host_path: asset_volume_directory(version), container_path: asset_path
     end
   end
 
-  def asset_extracted_path(version = nil)
-    File.join config.run_directory, "assets", "extracted", container_name(version)
+  def asset_extracted_directory(version = config.version)
+    File.join config.assets_directory, "extracted", [ name, version ].join("-")
   end
 
-  def asset_volume_path(version = nil)
-    File.join config.run_directory, "assets", "volumes", container_name(version)
+  def asset_volume_directory(version = config.version)
+    File.join config.assets_directory, "volumes", [ name, version ].join("-")
+  end
+
+  def ensure_one_host_for_ssl
+    if running_proxy? && proxy.ssl? && hosts.size > 1
+      raise Kamal::ConfigurationError, "SSL is only supported on a single server, found #{hosts.size} servers for role #{name}"
+    end
   end
 
   private
+    def initialize_specialized_proxy
+      proxy_specializations = specializations["proxy"]
+
+      if primary?
+        # only false means no proxy for non-primary roles
+        @running_proxy = proxy_specializations != false
+      else
+        # false and nil both mean no proxy for non-primary roles
+        @running_proxy = !!proxy_specializations
+      end
+
+      if running_proxy?
+        proxy_config = proxy_specializations == true || proxy_specializations.nil? ? {} : proxy_specializations
+
+        @specialized_proxy = Kamal::Configuration::Proxy.new \
+          config: config,
+          proxy_config: proxy_config,
+          context: "servers/#{name}/proxy"
+      end
+    end
+
     def tagged_hosts
       {}.tap do |tagged_hosts|
         extract_hosts_from_config.map do |host_config|
@@ -221,27 +211,6 @@ class Kamal::Configuration::Role
       end
     end
 
-    def traefik_labels
-      if running_traefik?
-        {
-          # Setting a service property ensures that the generated service name will be consistent between versions
-          "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
-
-          "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
-          "traefik.http.routers.#{traefik_service}.priority" => "2",
-          "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
-          "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
-          "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
-        }
-      else
-        {}
-      end
-    end
-
-    def traefik_service
-      container_prefix
-    end
-
     def custom_labels
       Hash.new.tap do |labels|
         labels.merge!(config.labels) if config.labels.present?

data/lib/kamal/configuration/validator/builder.rb

@@ -7,5 +7,7 @@ class Kamal::Configuration::Validator::Builder < Kamal::Configuration::Validator
     end
 
     error "Builder arch not set" unless config["arch"].present?
+
+    error "Cannot disable local builds, no remote is set" if config["local"] == false && config["remote"].blank?
   end
 end

data/lib/kamal/configuration/validator/proxy.rb

@@ -0,0 +1,11 @@
+class Kamal::Configuration::Validator::Proxy < Kamal::Configuration::Validator
+  def validate!
+    unless config.nil?
+      super
+
+      if config["host"].blank? && config["ssl"]
+        error "Must set a host to enable automatic SSL"
+      end
+    end
+  end
+end

data/lib/kamal/configuration/validator.rb

@@ -24,7 +24,9 @@ class Kamal::Configuration::Validator
             example_value = example[key]
 
             if example_value == "..."
-              validate_type! value, *(Array if key == :servers), Hash
+              unless key.to_s == "proxy" && boolean?(value.class)
+                validate_type! value, *(Array if key == :servers), Hash
+              end
             elsif key == "hosts"
               validate_servers! value
             elsif example_value.is_a?(Array)