kamal 2.0.0.alpha → 2.0.0.beta2

data/lib/kamal/configuration.rb

@@ -2,19 +2,22 @@ require "active_support/ordered_options"
 require "active_support/core_ext/string/inquiry"
 require "active_support/core_ext/module/delegation"
 require "active_support/core_ext/hash/keys"
-require "pathname"
 require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :labels, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
+  delegate :service, :image, :labels, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_reader :destination, :raw_config
-  attr_reader :accessories, :aliases, :boot, :builder, :env, :healthcheck, :logging, :traefik, :servers, :ssh, :sshkit, :registry
+  attr_reader :destination, :raw_config, :secrets
+  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :proxy, :servers, :ssh, :sshkit, :registry
 
   include Validation
 
+  PROXY_MINIMUM_VERSION = "v0.4.0"
+  PROXY_HTTP_PORT = 80
+  PROXY_HTTPS_PORT = 443
+
   class << self
     def create_from(config_file:, destination: nil, version: nil)
       raw_config = load_config_files(config_file, *destination_config_file(config_file, destination))
@@ -49,6 +52,8 @@ class Kamal::Configuration
 
     validate! raw_config, example: validation_yml.symbolize_keys, context: "", with: Kamal::Configuration::Validator::Configuration
 
+    @secrets = Kamal::Secrets.new(destination: destination)
+
     # Eager load config to validate it, these are first as they have dependencies later on
     @servers = Servers.new(config: self)
     @registry = Registry.new(config: self)
@@ -57,11 +62,10 @@ class Kamal::Configuration
     @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
     @boot = Boot.new(config: self)
     @builder = Builder.new(config: self)
-    @env = Env.new(config: @raw_config.env || {})
+    @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
 
-    @healthcheck = Healthcheck.new(healthcheck_config: @raw_config.healthcheck)
     @logging = Logging.new(logging_config: @raw_config.logging)
-    @traefik = Traefik.new(config: self)
+    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy || {})
     @ssh = Ssh.new(config: self)
     @sshkit = Sshkit.new(config: self)
 
@@ -70,6 +74,9 @@ class Kamal::Configuration
     ensure_valid_kamal_version
     ensure_retain_containers_valid
     ensure_valid_service_name
+    ensure_no_traefik_reboot_hooks
+    ensure_one_host_for_ssl_roles
+    ensure_unique_hosts_for_ssl_roles
   end
 
 
@@ -130,16 +137,16 @@ class Kamal::Configuration
     raw_config.allow_empty_roles
   end
 
-  def traefik_roles
-    roles.select(&:running_traefik?)
+  def proxy_roles
+    roles.select(&:running_proxy?)
   end
 
-  def traefik_role_names
-    traefik_roles.flat_map(&:name)
+  def proxy_role_names
+    proxy_roles.flat_map(&:name)
   end
 
-  def traefik_hosts
-    traefik_roles.flat_map(&:hosts).uniq
+  def proxy_hosts
+    proxy_roles.flat_map(&:hosts).uniq
   end
 
   def repository
@@ -184,31 +191,40 @@ class Kamal::Configuration
   end
 
 
-  def healthcheck_service
-    [ "healthcheck", service, destination ].compact.join("-")
-  end
-
   def readiness_delay
     raw_config.readiness_delay || 7
   end
 
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
+  def deploy_timeout
+    raw_config.deploy_timeout || 30
+  end
+
+  def drain_timeout
+    raw_config.drain_timeout || 30
   end
 
 
   def run_directory
-    raw_config.run_directory || ".kamal"
+    ".kamal"
   end
 
-  def run_directory_as_docker_volume
-    if Pathname.new(run_directory).absolute?
-      run_directory
-    else
-      File.join "$(pwd)", run_directory
-    end
+  def apps_directory
+    File.join run_directory, "apps"
+  end
+
+  def app_directory
+    File.join apps_directory, [ service, destination ].compact.join("-")
   end
 
+  def env_directory
+    File.join app_directory, "env"
+  end
+
+  def assets_directory
+    File.join app_directory, "assets"
+  end
+
+
   def hooks_path
     raw_config.hooks_path || ".kamal/hooks"
   end
@@ -218,13 +234,9 @@ class Kamal::Configuration
   end
 
 
-  def host_env_directory
-    File.join(run_directory, "env")
-  end
-
   def env_tags
     @env_tags ||= if (tags = raw_config.env["tags"])
-      tags.collect { |name, config| Env::Tag.new(name, config: config) }
+      tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
     else
       []
     end
@@ -234,6 +246,18 @@ class Kamal::Configuration
     env_tags.detect { |t| t.name == name.to_s }
   end
 
+  def proxy_publish_args
+    argumentize "--publish", [ "#{PROXY_HTTP_PORT}:#{PROXY_HTTP_PORT}", "#{PROXY_HTTPS_PORT}:#{PROXY_HTTPS_PORT}" ]
+  end
+
+  def proxy_image
+    "basecamp/kamal-proxy:#{PROXY_MINIMUM_VERSION}"
+  end
+
+  def proxy_container_name
+    "kamal-proxy"
+  end
+
 
   def to_h
     {
@@ -249,8 +273,7 @@ class Kamal::Configuration
       sshkit: sshkit.to_h,
       builder: builder.to_h,
       accessories: raw_config.accessories,
-      logging: logging_args,
-      healthcheck: healthcheck.to_h
+      logging: logging_args
     }.compact
   end
 
@@ -308,6 +331,30 @@ class Kamal::Configuration
       true
     end
 
+    def ensure_no_traefik_reboot_hooks
+      hooks = %w[ pre-traefik-reboot post-traefik-reboot ].select { |hook_file| File.exist?(File.join(hooks_path, hook_file)) }
+
+      if hooks.any?
+        raise Kamal::ConfigurationError, "Found #{hooks.join(", ")}, these should be renamed to (pre|post)-proxy-reboot"
+      end
+
+      true
+    end
+
+    def ensure_one_host_for_ssl_roles
+      roles.each(&:ensure_one_host_for_ssl)
+
+      true
+    end
+
+    def ensure_unique_hosts_for_ssl_roles
+      hosts = roles.select(&:ssl?).map { |role| role.proxy.host }
+      duplicates = hosts.tally.filter_map { |host, count| host if count > 1 }
+
+      raise Kamal::ConfigurationError, "Different roles can't share the same host for SSL: #{duplicates.join(", ")}" if duplicates.any?
+
+      true
+    end
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort

data/lib/kamal/env_file.rb

@@ -15,6 +15,10 @@ class Kamal::EnvFile
     env_file.presence || "\n"
   end
 
+  def to_io
+    StringIO.new(to_s)
+  end
+
   alias to_str to_s
 
   private

data/lib/kamal/secrets/adapters/base.rb

@@ -0,0 +1,18 @@
+class Kamal::Secrets::Adapters::Base
+  delegate :optionize, to: Kamal::Utils
+
+  def fetch(secrets, account:, from: nil)
+    session = login(account)
+    full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") }
+    fetch_secrets(full_secrets, account: account, session: session)
+  end
+
+  private
+    def login(...)
+      raise NotImplementedError
+    end
+
+    def fetch_secrets(...)
+      raise NotImplementedError
+    end
+end

data/lib/kamal/secrets/adapters/bitwarden.rb

@@ -0,0 +1,64 @@
+class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base
+  private
+    def login(account)
+      status = run_command("status")
+
+      if status["status"] == "unauthenticated"
+        run_command("login #{account.shellescape}", raw: true)
+        status = run_command("status")
+      end
+
+      if status["status"] == "locked"
+        session = run_command("unlock --raw", raw: true).presence
+        status = run_command("status", session: session)
+      end
+
+      raise RuntimeError, "Failed to login to and unlock Bitwarden" unless status["status"] == "unlocked"
+
+      run_command("sync", session: session, raw: true)
+      raise RuntimeError, "Failed to sync Bitwarden" unless $?.success?
+
+      session
+    end
+
+    def fetch_secrets(secrets, account:, session:)
+      {}.tap do |results|
+        items_fields(secrets).each do |item, fields|
+          item_json = run_command("get item #{item.shellescape}", session: session, raw: true)
+          raise RuntimeError, "Could not read #{secret} from Bitwarden" unless $?.success?
+          item_json = JSON.parse(item_json)
+
+          if fields.any?
+            fields.each do |field|
+              item_field = item_json["fields"].find { |f| f["name"] == field }
+              raise RuntimeError, "Could not find field #{field} in item #{item} in Bitwarden" unless item_field
+              value = item_field["value"]
+              results["#{item}/#{field}"] = value
+            end
+          else
+            results[item] = item_json["login"]["password"]
+          end
+        end
+      end
+    end
+
+    def items_fields(secrets)
+      {}.tap do |items|
+        secrets.each do |secret|
+          item, field = secret.split("/")
+          items[item] ||= []
+          items[item] << field
+        end
+      end
+    end
+
+    def signedin?(account)
+      run_command("status")["status"] != "unauthenticated"
+    end
+
+    def run_command(command, session: nil, raw: false)
+      full_command = [ *("BW_SESSION=#{session.shellescape}" if session), "bw", command ].join(" ")
+      result = `#{full_command}`.strip
+      raw ? result : JSON.parse(result)
+    end
+end

data/lib/kamal/secrets/adapters/last_pass.rb

@@ -0,0 +1,30 @@
+class Kamal::Secrets::Adapters::LastPass < Kamal::Secrets::Adapters::Base
+  private
+    def login(account)
+      unless loggedin?(account)
+        `lpass login #{account.shellescape}`
+        raise RuntimeError, "Failed to login to 1Password" unless $?.success?
+      end
+    end
+
+    def loggedin?(account)
+      `lpass status --color never`.strip == "Logged in as #{account}."
+    end
+
+    def fetch_secrets(secrets, account:, session:)
+      items = `lpass show #{secrets.map(&:shellescape).join(" ")} --json`
+      raise RuntimeError, "Could not read #{secrets} from 1Password" unless $?.success?
+
+      items = JSON.parse(items)
+
+      {}.tap do |results|
+        items.each do |item|
+          results[item["fullname"]] = item["password"]
+        end
+
+        if (missing_items = secrets - results.keys).any?
+          raise RuntimeError, "Could not find #{missing_items.join(", ")} in LassPass"
+        end
+      end
+    end
+end

data/lib/kamal/secrets/adapters/one_password.rb

@@ -0,0 +1,61 @@
+class Kamal::Secrets::Adapters::OnePassword < Kamal::Secrets::Adapters::Base
+  delegate :optionize, to: Kamal::Utils
+
+  private
+    def login(account)
+      unless loggedin?(account)
+        `op signin #{to_options(account: account, force: true, raw: true)}`.tap do
+          raise RuntimeError, "Failed to login to 1Password" unless $?.success?
+        end
+      end
+    end
+
+    def loggedin?(account)
+      `op account get --account #{account.shellescape} 2> /dev/null`
+      $?.success?
+    end
+
+    def fetch_secrets(secrets, account:, session:)
+      {}.tap do |results|
+        vaults_items_fields(secrets).map do |vault, items|
+          items.each do |item, fields|
+            fields_json = JSON.parse(op_item_get(vault, item, fields, account: account, session: session))
+            fields_json = [ fields_json ] if fields.one?
+
+            fields_json.each do |field_json|
+              # The reference is in the form `op://vault/item/field[/field]`
+              field = field_json["reference"].delete_prefix("op://").delete_suffix("/password")
+              results[field] = field_json["value"]
+            end
+          end
+        end
+      end
+    end
+
+    def to_options(**options)
+      optionize(options.compact).join(" ")
+    end
+
+    def vaults_items_fields(secrets)
+      {}.tap do |vaults|
+        secrets.each do |secret|
+          secret = secret.delete_prefix("op://")
+          vault, item, *fields = secret.split("/")
+          fields << "password" if fields.empty?
+
+          vaults[vault] ||= {}
+          vaults[vault][item] ||= []
+          vaults[vault][item] << fields.join(".")
+        end
+      end
+    end
+
+    def op_item_get(vault, item, fields, account:, session:)
+      labels = fields.map { |field| "label=#{field}" }.join(",")
+      options = to_options(vault: vault, fields: labels, format: "json", account: account, session: session.presence)
+
+      `op item get #{item.shellescape} #{options}`.tap do
+        raise RuntimeError, "Could not read #{fields.join(", ")} from #{item} in the #{vault} 1Password vault" unless $?.success?
+      end
+    end
+end

data/lib/kamal/secrets/adapters/test.rb

@@ -0,0 +1,10 @@
+class Kamal::Secrets::Adapters::Test < Kamal::Secrets::Adapters::Base
+  private
+    def login(account)
+      true
+    end
+
+    def fetch_secrets(secrets, account:, session:)
+      secrets.to_h { |secret| [ secret, secret.reverse ] }
+    end
+end

data/lib/kamal/secrets/adapters.rb

@@ -0,0 +1,14 @@
+require "active_support/core_ext/string/inflections"
+module Kamal::Secrets::Adapters
+  def self.lookup(name)
+    name = "one_password" if name.downcase == "1password"
+    name = "last_pass" if name.downcase == "lastpass"
+    adapter_class(name)
+  end
+
+  def self.adapter_class(name)
+    Object.const_get("Kamal::Secrets::Adapters::#{name.camelize}").new
+  rescue NameError => e
+    raise RuntimeError, "Unknown secrets adapter: #{name}"
+  end
+end

data/lib/kamal/secrets/dotenv/inline_command_substitution.rb

@@ -0,0 +1,32 @@
+class Kamal::Secrets::Dotenv::InlineCommandSubstitution
+  class << self
+    def install!
+      ::Dotenv::Parser.substitutions.map! { |sub| sub == ::Dotenv::Substitutions::Command ? self : sub }
+    end
+
+    def call(value, _env, overwrite: false)
+      # Process interpolated shell commands
+      value.gsub(Dotenv::Substitutions::Command.singleton_class::INTERPOLATED_SHELL_COMMAND) do |*|
+        # Eliminate opening and closing parentheses
+        command = $LAST_MATCH_INFO[:cmd][1..-2]
+
+        if $LAST_MATCH_INFO[:backslash]
+          # Command is escaped, don't replace it.
+          $LAST_MATCH_INFO[0][1..]
+        else
+          if command =~ /\A\s*kamal\s*secrets\s+/
+            # Inline the command
+            inline_secrets_command(command)
+          else
+            # Execute the command and return the value
+            `#{command}`.chomp
+          end
+        end
+      end
+    end
+
+    def inline_secrets_command(command)
+      Kamal::Cli::Main.start(command.shellsplit[1..] + [ "--inline" ]).chomp
+    end
+  end
+end

data/lib/kamal/secrets.rb

@@ -0,0 +1,37 @@
+require "dotenv"
+
+class Kamal::Secrets
+  attr_reader :secrets_files
+
+  Kamal::Secrets::Dotenv::InlineCommandSubstitution.install!
+
+  def initialize(destination: nil)
+    @secrets_files = \
+      [ ".kamal/secrets-common", ".kamal/secrets#{(".#{destination}" if destination)}" ].select { |f| File.exist?(f) }
+    @mutex = Mutex.new
+  end
+
+  def [](key)
+    # Fetching secrets may ask the user for input, so ensure only one thread does that
+    @mutex.synchronize do
+      secrets.fetch(key)
+    end
+  rescue KeyError
+    if secrets_files
+      raise Kamal::ConfigurationError, "Secret '#{key}' not found in #{secrets_files.join(", ")}"
+    else
+      raise Kamal::ConfigurationError, "Secret '#{key}' not found, no secret files provided"
+    end
+  end
+
+  def to_h
+    secrets
+  end
+
+  private
+    def secrets
+      @secrets ||= secrets_files.inject({}) do |secrets, secrets_file|
+        secrets.merge!(::Dotenv.parse(secrets_file))
+      end
+    end
+end

data/lib/kamal/sshkit_with_ext.rb

@@ -3,6 +3,7 @@ require "sshkit/dsl"
 require "net/scp"
 require "active_support/core_ext/hash/deep_merge"
 require "json"
+require "concurrent/atomic/semaphore"
 
 class SSHKit::Backend::Abstract
   def capture_with_info(*args, **kwargs)

data/lib/kamal/utils.rb

@@ -1,3 +1,5 @@
+require "active_support/core_ext/object/try"
+
 module Kamal::Utils
   extend self
 
@@ -54,6 +56,12 @@ module Kamal::Utils
 
   # Escape a value to make it safe for shell use.
   def escape_shell_value(value)
+    value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/) \
+      .map { |part| part.ascii_only? ? escape_ascii_shell_value(part) : part }
+      .join
+  end
+
+  def escape_ascii_shell_value(value)
     value.to_s.dump
       .gsub(/`/, '\\\\`')
       .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
@@ -93,4 +101,8 @@ module Kamal::Utils
       arch
     end
   end
+
+  def older_version?(version, other_version)
+    Gem::Version.new(version.delete_prefix("v")) < Gem::Version.new(other_version.delete_prefix("v"))
+  end
 end

data/lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "2.0.0.alpha"
+  VERSION = "2.0.0.beta2"
 end

data/lib/kamal.rb

@@ -5,8 +5,10 @@ end
 require "active_support"
 require "zeitwerk"
 require "yaml"
+require "tmpdir"
+require "pathname"
 
 loader = Zeitwerk::Loader.for_gem
 loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
 loader.setup
-loader.eager_load # We need all commands loaded.
+loader.eager_load_namespace(Kamal::Cli) # We need all commands loaded.