kamal 2.4.0 → 2.5.0

data/lib/kamal/configuration/accessory.rb

@@ -5,7 +5,7 @@ class Kamal::Configuration::Accessory
 
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_reader :name, :accessory_config, :env, :proxy
+  attr_reader :name, :env, :proxy, :registry
 
   def initialize(name, config:)
     @name, @config, @accessory_config = name.inquiry, config, config.raw_config["accessories"][name]
@@ -16,12 +16,11 @@ class Kamal::Configuration::Accessory
       context: "accessories/#{name}",
       with: Kamal::Configuration::Validator::Accessory
 
-    @env = Kamal::Configuration::Env.new \
-      config: accessory_config.fetch("env", {}),
-      secrets: config.secrets,
-      context: "accessories/#{name}/env"
+    ensure_valid_roles
 
-    initialize_proxy if running_proxy?
+    @env = initialize_env
+    @proxy = initialize_proxy if running_proxy?
+    @registry = initialize_registry if accessory_config["registry"].present?
   end
 
   def service_name
@@ -29,7 +28,7 @@ class Kamal::Configuration::Accessory
   end
 
   def image
-    accessory_config["image"]
+    [ registry&.server, accessory_config["image"] ].compact.join("/")
   end
 
   def hosts
@@ -109,18 +108,32 @@ class Kamal::Configuration::Accessory
   end
 
   def running_proxy?
-    @accessory_config["proxy"].present?
-  end
-
-  def initialize_proxy
-    @proxy = Kamal::Configuration::Proxy.new \
-      config: config,
-      proxy_config: accessory_config["proxy"],
-      context: "accessories/#{name}/proxy"
+    accessory_config["proxy"].present?
   end
 
   private
-    attr_accessor :config
+    attr_reader :config, :accessory_config
+
+    def initialize_env
+      Kamal::Configuration::Env.new \
+        config: accessory_config.fetch("env", {}),
+        secrets: config.secrets,
+        context: "accessories/#{name}/env"
+    end
+
+    def initialize_proxy
+      Kamal::Configuration::Proxy.new \
+        config: config,
+        proxy_config: accessory_config["proxy"],
+        context: "accessories/#{name}/proxy"
+    end
+
+    def initialize_registry
+      Kamal::Configuration::Registry.new \
+        config: accessory_config,
+        secrets: config.secrets,
+        context: "accessories/#{name}/registry"
+    end
 
     def default_labels
       { "service" => service_name }
@@ -189,13 +202,17 @@ class Kamal::Configuration::Accessory
 
     def hosts_from_roles
       if accessory_config.key?("roles")
-        accessory_config["roles"].flat_map do |role|
-          config.role(role)&.hosts || raise(Kamal::ConfigurationError, "Unknown role in accessories config: '#{role}'")
-        end
+        accessory_config["roles"].flat_map { |role| config.role(role)&.hosts }
       end
     end
 
     def network
       accessory_config["network"] || DEFAULT_NETWORK
     end
+
+    def ensure_valid_roles
+      if accessory_config["roles"] && (missing_roles = accessory_config["roles"] - config.roles.map(&:name)).any?
+        raise Kamal::ConfigurationError, "accessories/#{name}: unknown roles #{missing_roles.join(", ")}"
+      end
+    end
 end

data/lib/kamal/configuration/builder.rb

@@ -53,6 +53,10 @@ class Kamal::Configuration::Builder
     !local_disabled? && (arches.empty? || local_arches.any?)
   end
 
+  def cloud?
+    driver.start_with? "cloud"
+  end
+
   def cached?
     !!builder_config["cache"]
   end

data/lib/kamal/configuration/docs/accessory.yml

@@ -23,9 +23,27 @@ accessories:
 
     # Image
     #
-    # The Docker image to use, prefix it with a registry if not using Docker Hub:
+    # The Docker image to use.
+    # Prefix it with its server when using root level registry different from Docker Hub.
+    # Define registry directly or via anchors when it differs from root level registry.
     image: mysql:8.0
 
+    # Registry
+    #
+    # By default accessories use Docker Hub registry.
+    # You can specify different registry per accessory with this option.
+    # Don't prefix image with this registry server.
+    # Use anchors if you need to set the same specific registry for several accessories.
+    #
+    # ```yml
+    # registry:
+    #   <<: *specific-registry
+    # ```
+    #
+    # See kamal docs registry for more information:
+    registry:
+      ...
+
     # Accessory hosts
     #
     # Specify one of `host`, `hosts`, or `roles`:
@@ -100,5 +118,6 @@ accessories:
 
     # Proxy
     #
+    # You can run your accessory behind the Kamal proxy. See kamal docs proxy for more information
     proxy:
       ...

data/lib/kamal/configuration/docs/builder.yml

@@ -102,6 +102,9 @@ builder:
   #
   # The build driver to use, defaults to `docker-container`:
   driver: docker
+  #
+  # If you want to use Docker Build Cloud (https://www.docker.com/products/build-cloud/), you can set the driver to:
+  driver: cloud org-name/builder-name
 
   # Provenance
   #

data/lib/kamal/configuration/registry.rb

@@ -1,12 +1,10 @@
 class Kamal::Configuration::Registry
   include Kamal::Configuration::Validation
 
-  attr_reader :registry_config, :secrets
-
-  def initialize(config:)
-    @registry_config = config.raw_config.registry || {}
-    @secrets = config.secrets
-    validate! registry_config, with: Kamal::Configuration::Validator::Registry
+  def initialize(config:, secrets:, context: "registry")
+    @registry_config = config["registry"] || {}
+    @secrets = secrets
+    validate! registry_config, context: context, with: Kamal::Configuration::Validator::Registry
   end
 
   def server
@@ -22,6 +20,8 @@ class Kamal::Configuration::Registry
   end
 
   private
+    attr_reader :registry_config, :secrets
+
     def lookup(key)
       if registry_config[key].is_a?(Array)
         secrets[registry_config[key].first]

data/lib/kamal/configuration/role.rb

@@ -10,7 +10,7 @@ class Kamal::Configuration::Role
   def initialize(name, config:)
     @name, @config = name.inquiry, config
     validate! \
-      specializations,
+      role_config,
       example: validation_yml["servers"]["workers"],
       context: "servers/#{name}",
       with: Kamal::Configuration::Validator::Role
@@ -204,11 +204,11 @@ class Kamal::Configuration::Role
     end
 
     def specializations
-      if config.raw_config.servers.is_a?(Array) || config.raw_config.servers[name].is_a?(Array)
-        {}
-      else
-        config.raw_config.servers[name]
-      end
+      @specializations ||= role_config.is_a?(Array) ? {} : role_config
+    end
+
+    def role_config
+      @role_config ||= config.raw_config.servers.is_a?(Array) ? {} : config.raw_config.servers[name]
     end
 
     def custom_labels

data/lib/kamal/configuration/validator/role.rb

@@ -3,7 +3,7 @@ class Kamal::Configuration::Validator::Role < Kamal::Configuration::Validator
     validate_type! config, Array, Hash
 
     if config.is_a?(Array)
-      validate_servers! "servers", config
+      validate_servers!(config)
     else
       super
     end

data/lib/kamal/configuration.rb

@@ -59,7 +59,7 @@ class Kamal::Configuration
 
     # Eager load config to validate it, these are first as they have dependencies later on
     @servers = Servers.new(config: self)
-    @registry = Registry.new(config: self)
+    @registry = Registry.new(config: @raw_config, secrets: secrets)
 
     @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || []
     @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
@@ -82,7 +82,6 @@ class Kamal::Configuration
     ensure_unique_hosts_for_ssl_roles
   end
 
-
   def version=(version)
     @declared_version = version
   end
@@ -106,7 +105,6 @@ class Kamal::Configuration
     raw_config.minimum_version
   end
 
-
   def roles
     servers.roles
   end
@@ -119,7 +117,6 @@ class Kamal::Configuration
     accessories.detect { |a| a.name == name.to_s }
   end
 
-
   def all_hosts
     (roles + accessories).flat_map(&:hosts).uniq
   end
@@ -180,7 +177,6 @@ class Kamal::Configuration
     raw_config.retain_containers || 5
   end
 
-
   def volume_args
     if raw_config.volumes.present?
       argumentize "--volume", raw_config.volumes
@@ -193,7 +189,6 @@ class Kamal::Configuration
     logging.args
   end
 
-
   def readiness_delay
     raw_config.readiness_delay || 7
   end
@@ -206,7 +201,6 @@ class Kamal::Configuration
     raw_config.drain_timeout || 30
   end
 
-
   def run_directory
     ".kamal"
   end
@@ -227,7 +221,6 @@ class Kamal::Configuration
     File.join app_directory, "assets"
   end
 
-
   def hooks_path
     raw_config.hooks_path || ".kamal/hooks"
   end
@@ -236,7 +229,6 @@ class Kamal::Configuration
     raw_config.asset_path
   end
 
-
   def env_tags
     @env_tags ||= if (tags = raw_config.env["tags"])
       tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
@@ -249,8 +241,16 @@ class Kamal::Configuration
     env_tags.detect { |t| t.name == name.to_s }
   end
 
-  def proxy_publish_args(http_port, https_port)
-    argumentize "--publish", [ "#{http_port}:#{PROXY_HTTP_PORT}", "#{https_port}:#{PROXY_HTTPS_PORT}" ]
+  def proxy_publish_args(http_port, https_port, bind_ips = nil)
+    ensure_valid_bind_ips(bind_ips)
+
+    (bind_ips || [ nil ]).map do |bind_ip|
+      bind_ip = format_bind_ip(bind_ip)
+      publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":")
+      publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":")
+
+      argumentize "--publish", [ publish_http, publish_https ]
+    end.join(" ")
   end
 
   def proxy_logging_args(max_size)
@@ -277,7 +277,6 @@ class Kamal::Configuration
     File.join proxy_directory, "options"
   end
 
-
   def to_h
     {
       roles: role_names,
@@ -344,6 +343,15 @@ class Kamal::Configuration
       true
     end
 
+    def ensure_valid_bind_ips(bind_ips)
+      bind_ips.present? && bind_ips.each do |ip|
+        next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex
+        raise ArgumentError, "Invalid publish IP address: #{ip}"
+      end
+
+      true
+    end
+
     def ensure_retain_containers_valid
       raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1
 
@@ -375,6 +383,15 @@ class Kamal::Configuration
       true
     end
 
+    def format_bind_ip(ip)
+      # Ensure IPv6 address inside square brackets - e.g. [::1]
+      if ip =~ Resolv::IPv6::Regex && ip !~ /\[.*\]/
+        "[#{ip}]"
+      else
+        ip
+      end
+    end
+
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
     end

data/lib/kamal/docker.rb

@@ -0,0 +1,30 @@
+require "tempfile"
+require "open3"
+
+module Kamal::Docker
+  extend self
+  BUILD_CHECK_TAG = "kamal-local-build-check"
+
+  def included_files
+    Tempfile.create do |dockerfile|
+      dockerfile.write(<<~DOCKERFILE)
+        FROM busybox
+        COPY . app
+        WORKDIR app
+        CMD find . -type f | sed "s|^\./||"
+      DOCKERFILE
+      dockerfile.close
+
+      cmd = "docker buildx build -t=#{BUILD_CHECK_TAG} -f=#{dockerfile.path} ."
+      system(cmd) || raise("failed to build check image")
+    end
+
+    cmd = "docker run --rm #{BUILD_CHECK_TAG}"
+    out, err, status = Open3.capture3(cmd)
+    unless status
+      raise "failed to run check image:\n#{err}"
+    end
+
+    out.lines.map(&:strip)
+  end
+end

data/lib/kamal/git.rb

@@ -24,4 +24,14 @@ module Kamal::Git
   def root
     `git rev-parse --show-toplevel`.strip
   end
+
+  # returns an array of relative path names of files with uncommitted changes
+  def uncommitted_files
+    `git ls-files --modified`.lines.map(&:strip)
+  end
+
+  # returns an array of relative path names of untracked files, including gitignored files
+  def untracked_files
+    `git ls-files --others`.lines.map(&:strip)
+  end
 end

data/lib/kamal/secrets/adapters/aws_secrets_manager.rb

@@ -1,12 +1,16 @@
 class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
   private
     def login(_account)
       nil
     end
 
-    def fetch_secrets(secrets, account:, session:)
+    def fetch_secrets(secrets, from:, account: nil, session:)
       {}.tap do |results|
-        get_from_secrets_manager(secrets, account: account).each do |secret|
+        get_from_secrets_manager(prefixed_secrets(secrets, from: from), account: account).each do |secret|
           secret_name = secret["Name"]
           secret_string = JSON.parse(secret["SecretString"])
 
@@ -19,8 +23,12 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba
       end
     end
 
-    def get_from_secrets_manager(secrets, account:)
-      `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account.shellescape}`.tap do |secrets|
+    def get_from_secrets_manager(secrets, account: nil)
+      args = [ "aws", "secretsmanager", "batch-get-secret-value", "--secret-id-list" ] + secrets.map(&:shellescape)
+      args += [ "--profile", account.shellescape ] if account
+      cmd = args.join(" ")
+
+      `#{cmd}`.tap do |secrets|
         raise RuntimeError, "Could not read #{secrets} from AWS Secrets Manager" unless $?.success?
 
         secrets = JSON.parse(secrets)

data/lib/kamal/secrets/adapters/base.rb

@@ -7,8 +7,7 @@ class Kamal::Secrets::Adapters::Base
     check_dependencies!
 
     session = login(account)
-    full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") }
-    fetch_secrets(full_secrets, account: account, session: session)
+    fetch_secrets(secrets, from: from, account: account, session: session)
   end
 
   def requires_account?
@@ -27,4 +26,8 @@ class Kamal::Secrets::Adapters::Base
     def check_dependencies!
       raise NotImplementedError
     end
+
+    def prefixed_secrets(secrets, from:)
+      secrets.map { |secret| [ from, secret ].compact.join("/") }
+    end
 end

data/lib/kamal/secrets/adapters/bitwarden.rb

@@ -21,9 +21,9 @@ class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base
       session
     end
 
-    def fetch_secrets(secrets, account:, session:)
+    def fetch_secrets(secrets, from:, account:, session:)
       {}.tap do |results|
-        items_fields(secrets).each do |item, fields|
+        items_fields(prefixed_secrets(secrets, from: from)).each do |item, fields|
           item_json = run_command("get item #{item.shellescape}", session: session, raw: true)
           raise RuntimeError, "Could not read #{item} from Bitwarden" unless $?.success?
           item_json = JSON.parse(item_json)

data/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb

@@ -0,0 +1,72 @@
+class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    LIST_ALL_SELECTOR = "all"
+    LIST_ALL_FROM_PROJECT_SUFFIX = "/all"
+    LIST_COMMAND = "secret list -o env"
+    GET_COMMAND = "secret get -o env"
+
+    def fetch_secrets(secrets, from:, account:, session:)
+      raise RuntimeError, "You must specify what to retrieve from Bitwarden Secrets Manager" if secrets.length == 0
+
+      secrets = prefixed_secrets(secrets, from: from)
+      command, project = extract_command_and_project(secrets)
+
+      {}.tap do |results|
+        if command.nil?
+          secrets.each do |secret_uuid|
+            secret = run_command("#{GET_COMMAND} #{secret_uuid.shellescape}")
+            raise RuntimeError, "Could not read #{secret_uuid} from Bitwarden Secrets Manager" unless $?.success?
+            key, value = parse_secret(secret)
+            results[key] = value
+          end
+        else
+          secrets = run_command(command)
+          raise RuntimeError, "Could not read secrets from Bitwarden Secrets Manager" unless $?.success?
+          secrets.split("\n").each do |secret|
+            key, value = parse_secret(secret)
+            results[key] = value
+          end
+        end
+      end
+    end
+
+    def extract_command_and_project(secrets)
+      if secrets.length == 1
+        if secrets[0] == LIST_ALL_SELECTOR
+          [ LIST_COMMAND, nil ]
+        elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX)
+          project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first
+          [ "#{LIST_COMMAND} #{project.shellescape}", project ]
+        end
+      end
+    end
+
+    def parse_secret(secret)
+      key, value = secret.split("=", 2)
+      value = value.gsub(/^"|"$/, "")
+      [ key, value ]
+    end
+
+    def run_command(command, session: nil)
+      full_command = [ "bws", command ].join(" ")
+      `#{full_command}`
+    end
+
+    def login(account)
+      run_command("run 'echo OK'")
+      raise RuntimeError, "Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?" unless $?.success?
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Bitwarden Secrets Manager CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `bws --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/doppler.rb

@@ -16,8 +16,21 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base
       $?.success?
     end
 
-    def fetch_secrets(secrets, **)
-      project_and_config_flags = ""
+    def fetch_secrets(secrets, from:, **)
+      secrets = prefixed_secrets(secrets, from: from)
+      flags = secrets_get_flags(secrets)
+
+      secret_names = secrets.collect { |s| s.split("/").last }
+
+      items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{flags}`
+      raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success?
+
+      items = JSON.parse(items)
+
+      items.transform_values { |value| value["computed"] }
+    end
+
+    def secrets_get_flags(secrets)
       unless service_token_set?
         project, config, _ = secrets.first.split("/")
 
@@ -27,15 +40,6 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base
 
         project_and_config_flags = "-p #{project.shellescape} -c #{config.shellescape}"
       end
-
-      secret_names = secrets.collect { |s| s.split("/").last }
-
-      items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{project_and_config_flags}`
-      raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success?
-
-      items = JSON.parse(items)
-
-      items.transform_values { |value| value["computed"] }
     end
 
     def service_token_set?

data/lib/kamal/secrets/adapters/enpass.rb

@@ -0,0 +1,71 @@
+##
+# Enpass is different from most password managers, in a way that it's offline and doesn't need an account.
+#
+# Usage
+#
+# Fetch all password from FooBar item
+# `kamal secrets fetch --adapter enpass --from /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary FooBar`
+#
+# Fetch only DB_PASSWORD from FooBar item
+# `kamal secrets fetch --adapter enpass --from /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary FooBar/DB_PASSWORD`
+class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+    def fetch_secrets(secrets, from:, account:, session:)
+      secrets_titles = fetch_secret_titles(secrets)
+
+      result = `enpass-cli -json -vault #{from.shellescape} show #{secrets_titles.map(&:shellescape).join(" ")}`.strip
+
+      parse_result_and_take_secrets(result, secrets)
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Enpass CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `enpass-cli version 2> /dev/null`
+      $?.success?
+    end
+
+    def login(account)
+      nil
+    end
+
+    def fetch_secret_titles(secrets)
+      secrets.reduce(Set.new) do |secret_titles, secret|
+        # Sometimes secrets contain a '/', when the intent is to fetch a single password for an item. Example: FooBar/DB_PASSWORD
+        # Another case is, when the intent is to fetch all passwords for an item. Example: FooBar (and FooBar may have multiple different passwords)
+        key, separator, value = secret.rpartition("/")
+        if key.empty?
+          secret_titles << value
+        else
+          secret_titles << key
+        end
+      end.to_a
+    end
+
+    def parse_result_and_take_secrets(unparsed_result, secrets)
+      result = JSON.parse(unparsed_result)
+
+      result.reduce({}) do |secrets_with_passwords, item|
+        title = item["title"]
+        label = item["label"]
+        password = item["password"]
+
+        if title && password.present?
+          key = [ title, label ].compact.reject(&:empty?).join("/")
+
+          if secrets.include?(title) || secrets.include?(key)
+            raise RuntimeError, "#{key} is present more than once" if secrets_with_passwords[key]
+            secrets_with_passwords[key] = password
+          end
+        end
+
+        secrets_with_passwords
+      end
+    end
+end