kamal-insecure 2.7.0

data/lib/kamal/secrets/adapters/one_password.rb

@@ -0,0 +1,104 @@
+class Kamal::Secrets::Adapters::OnePassword < Kamal::Secrets::Adapters::Base
+  delegate :optionize, to: Kamal::Utils
+
+  private
+    def login(account)
+      unless loggedin?(account)
+        `op signin #{to_options(account: account, force: true, raw: true)}`.tap do
+          raise RuntimeError, "Failed to login to 1Password" unless $?.success?
+        end
+      end
+    end
+
+    def loggedin?(account)
+      `op account get --account #{account.shellescape} 2> /dev/null`
+      $?.success?
+    end
+
+    def fetch_secrets(secrets, from:, account:, session:)
+      if secrets.blank?
+        fetch_all_secrets(from: from, account: account, session: session)
+      else
+        fetch_specified_secrets(secrets, from: from, account: account, session: session)
+      end
+    end
+
+    def fetch_specified_secrets(secrets, from:, account:, session:)
+      {}.tap do |results|
+        vaults_items_fields(prefixed_secrets(secrets, from: from)).map do |vault, items|
+          items.each do |item, fields|
+            fields_json = JSON.parse(op_item_get(vault, item, fields: fields, account: account, session: session))
+            fields_json = [ fields_json ] if fields.one?
+
+            results.merge!(fields_map(fields_json))
+          end
+        end
+      end
+    end
+
+    def fetch_all_secrets(from:, account:, session:)
+      {}.tap do |results|
+        vault_items(from).each do |vault, items|
+          items.each do |item|
+            fields_json = JSON.parse(op_item_get(vault, item, account: account, session: session)).fetch("fields")
+
+            results.merge!(fields_map(fields_json))
+          end
+        end
+      end
+    end
+
+    def to_options(**options)
+      optionize(options.compact).join(" ")
+    end
+
+    def vaults_items_fields(secrets)
+      {}.tap do |vaults|
+        secrets.each do |secret|
+          secret = secret.delete_prefix("op://")
+          vault, item, *fields = secret.split("/")
+          fields << "password" if fields.empty?
+
+          vaults[vault] ||= {}
+          vaults[vault][item] ||= []
+          vaults[vault][item] << fields.join(".")
+        end
+      end
+    end
+
+    def vault_items(from)
+      from = from.delete_prefix("op://")
+      vault, item = from.split("/")
+      { vault => [ item ] }
+    end
+
+    def fields_map(fields_json)
+      fields_json.to_h do |field_json|
+        # The reference is in the form `op://vault/item/field[/field]`
+        field = field_json["reference"].delete_prefix("op://").delete_suffix("/password")
+        [ field, field_json["value"] ]
+      end
+    end
+
+    def op_item_get(vault, item, fields: nil, account:, session:)
+      options = { vault: vault, format: "json", account: account, session: session.presence }
+
+      if fields.present?
+        labels = fields.map { |field| "label=#{field}" }.join(",")
+        options.merge!(fields: labels)
+      end
+
+      `op item get #{item.shellescape} #{to_options(**options)}`.tap do
+        raise RuntimeError, "Could not read #{"#{fields.join(", ")} " if fields.present?}from #{item} in the #{vault} 1Password vault" unless $?.success?
+      end
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "1Password CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `op --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/passbolt.rb

@@ -0,0 +1,130 @@
+class Kamal::Secrets::Adapters::Passbolt < Kamal::Secrets::Adapters::Base
+  def requires_account?
+    false
+  end
+
+  private
+
+    def login(*)
+      `passbolt verify`
+      raise RuntimeError, "Failed to login to Passbolt" unless $?.success?
+    end
+
+    def fetch_secrets(secrets, from:, **)
+      secrets = prefixed_secrets(secrets, from: from)
+      raise ArgumentError, "No secrets given to fetch" if secrets.empty?
+
+      secret_names = secrets.collect { |s| s.split("/").last }
+      folders = secrets_get_folders(secrets)
+
+      # build filter conditions for each secret with its corresponding folder
+      filter_conditions = []
+      secrets.each do |secret|
+        parts = secret.split("/")
+        secret_name = parts.last
+
+        if parts.size > 1
+          # get the folder path without the secret name
+          folder_path = parts[0..-2]
+
+          # find the most nested folder for this path
+          current_folder = nil
+          current_path = []
+
+          folder_path.each do |folder_name|
+            current_path << folder_name
+            matching_folders = folders.select { |f| get_folder_path(f, folders) == current_path.join("/") }
+            current_folder = matching_folders.first if matching_folders.any?
+          end
+
+          if current_folder
+            filter_conditions << "(Name == #{secret_name.shellescape.inspect} && FolderParentID == #{current_folder["id"].shellescape.inspect})"
+          end
+        else
+          # for root level secrets (no folders)
+          filter_conditions << "Name == #{secret_name.shellescape.inspect}"
+        end
+      end
+
+      filter_condition = filter_conditions.any? ? "--filter '#{filter_conditions.join(" || ")}'" : ""
+      items = `passbolt list resources #{filter_condition} #{folders.map { |item| "--folder #{item["id"]}" }.join(" ")} --json`
+      raise RuntimeError, "Could not read #{secrets} from Passbolt" unless $?.success?
+
+      items = JSON.parse(items)
+      found_names = items.map { |item| item["name"] }
+      missing_secrets = secret_names - found_names
+      raise RuntimeError, "Could not find the following secrets in Passbolt: #{missing_secrets.join(", ")}" if missing_secrets.any?
+
+      items.to_h { |item| [ item["name"], item["password"] ] }
+    end
+
+    def secrets_get_folders(secrets)
+      # extract all folder paths (both parent and nested)
+      folder_paths = secrets
+        .select { |s| s.include?("/") }
+        .map { |s| s.split("/")[0..-2] } # get all parts except the secret name
+        .uniq
+
+      return [] if folder_paths.empty?
+
+      all_folders = []
+
+      # first get all top-level folders
+      parent_folders = folder_paths.map(&:first).uniq
+      filter_condition = "--filter '#{parent_folders.map { |name| "Name == #{name.shellescape.inspect}" }.join(" || ")}'"
+      fetch_folders = `passbolt list folders #{filter_condition} --json`
+      raise RuntimeError, "Could not read folders from Passbolt" unless $?.success?
+
+      parent_folder_items = JSON.parse(fetch_folders)
+      all_folders.concat(parent_folder_items)
+
+      # get nested folders for each parent
+      folder_paths.each do |path|
+        next if path.size <= 1 # skip non-nested folders
+
+        parent = path[0]
+        parent_folder = parent_folder_items.find { |f| f["name"] == parent }
+        next unless parent_folder
+
+        # for each nested level, get the folders using the parent's ID
+        current_parent = parent_folder
+        path[1..-1].each do |folder_name|
+          filter_condition = "--filter 'Name == #{folder_name.shellescape.inspect} && FolderParentID == #{current_parent["id"].shellescape.inspect}'"
+          fetch_nested = `passbolt list folders #{filter_condition} --json`
+          next unless $?.success?
+
+          nested_folders = JSON.parse(fetch_nested)
+          break if nested_folders.empty?
+
+          all_folders.concat(nested_folders)
+          current_parent = nested_folders.first
+        end
+      end
+
+      # check if we found all required folders
+      found_paths = all_folders.map { |f| get_folder_path(f, all_folders) }
+      missing_paths = folder_paths.map { |path| path.join("/") } - found_paths
+      raise RuntimeError, "Could not find the following folders in Passbolt: #{missing_paths.join(", ")}" if missing_paths.any?
+
+      all_folders
+    end
+
+    def get_folder_path(folder, all_folders, path = [])
+      path.unshift(folder["name"])
+      return path.join("/") if folder["folder_parent_id"].to_s.empty?
+
+      parent = all_folders.find { |f| f["id"] == folder["folder_parent_id"] }
+      return path.join("/") unless parent
+
+      get_folder_path(parent, all_folders, path)
+    end
+
+    def check_dependencies!
+      raise RuntimeError, "Passbolt CLI is not installed" unless cli_installed?
+    end
+
+    def cli_installed?
+      `passbolt --version 2> /dev/null`
+      $?.success?
+    end
+end

data/lib/kamal/secrets/adapters/test.rb

@@ -0,0 +1,14 @@
+class Kamal::Secrets::Adapters::Test < Kamal::Secrets::Adapters::Base
+  private
+    def login(account)
+      true
+    end
+
+    def fetch_secrets(secrets, from:, account:, session:)
+      prefixed_secrets(secrets, from: from).to_h { |secret| [ secret, secret.reverse ] }
+    end
+
+    def check_dependencies!
+      # no op
+    end
+end

data/lib/kamal/secrets/adapters.rb

@@ -0,0 +1,16 @@
+require "active_support/core_ext/string/inflections"
+module Kamal::Secrets::Adapters
+  def self.lookup(name)
+    name = "one_password" if name.downcase == "1password"
+    name = "last_pass" if name.downcase == "lastpass"
+    name = "gcp_secret_manager" if name.downcase == "gcp"
+    name = "bitwarden_secrets_manager" if name.downcase == "bitwarden-sm"
+    adapter_class(name)
+  end
+
+  def self.adapter_class(name)
+    Object.const_get("Kamal::Secrets::Adapters::#{name.camelize}").new
+  rescue NameError => e
+    raise RuntimeError, "Unknown secrets adapter: #{name}"
+  end
+end

data/lib/kamal/secrets/dotenv/inline_command_substitution.rb

@@ -0,0 +1,33 @@
+class Kamal::Secrets::Dotenv::InlineCommandSubstitution
+  class << self
+    def install!
+      ::Dotenv::Parser.substitutions.map! { |sub| sub == ::Dotenv::Substitutions::Command ? self : sub }
+    end
+
+    def call(value, env, overwrite: false)
+      # Process interpolated shell commands
+      value.gsub(Dotenv::Substitutions::Command.singleton_class::INTERPOLATED_SHELL_COMMAND) do |*|
+        # Eliminate opening and closing parentheses
+        command = $LAST_MATCH_INFO[:cmd][1..-2]
+
+        if $LAST_MATCH_INFO[:backslash]
+          # Command is escaped, don't replace it.
+          $LAST_MATCH_INFO[0][1..]
+        else
+          command = ::Dotenv::Substitutions::Variable.call(command, env)
+          if command =~ /\A\s*kamal\s*secrets\s+/
+            # Inline the command
+            inline_secrets_command(command)
+          else
+            # Execute the command and return the value
+            `#{command}`.chomp
+          end
+        end
+      end
+    end
+
+    def inline_secrets_command(command)
+      Kamal::Cli::Main.start(command.shellsplit[1..] + [ "--inline" ]).chomp
+    end
+  end
+end

data/lib/kamal/secrets.rb

@@ -0,0 +1,42 @@
+require "dotenv"
+
+class Kamal::Secrets
+  Kamal::Secrets::Dotenv::InlineCommandSubstitution.install!
+
+  def initialize(destination: nil)
+    @destination = destination
+    @mutex = Mutex.new
+  end
+
+  def [](key)
+    # Fetching secrets may ask the user for input, so ensure only one thread does that
+    @mutex.synchronize do
+      secrets.fetch(key)
+    end
+  rescue KeyError
+    if secrets_files.present?
+      raise Kamal::ConfigurationError, "Secret '#{key}' not found in #{secrets_files.join(", ")}"
+    else
+      raise Kamal::ConfigurationError, "Secret '#{key}' not found, no secret files (#{secrets_filenames.join(", ")}) provided"
+    end
+  end
+
+  def to_h
+    secrets
+  end
+
+  def secrets_files
+    @secrets_files ||= secrets_filenames.select { |f| File.exist?(f) }
+  end
+
+  private
+    def secrets
+      @secrets ||= secrets_files.inject({}) do |secrets, secrets_file|
+        secrets.merge!(::Dotenv.parse(secrets_file, overwrite: true))
+      end
+    end
+
+    def secrets_filenames
+      [ ".kamal/secrets-common", ".kamal/secrets#{(".#{@destination}" if @destination)}" ]
+    end
+end

data/lib/kamal/sshkit_with_ext.rb

@@ -0,0 +1,142 @@
+require "sshkit"
+require "sshkit/dsl"
+require "net/scp"
+require "active_support/core_ext/hash/deep_merge"
+require "json"
+require "concurrent/atomic/semaphore"
+
+class SSHKit::Backend::Abstract
+  def capture_with_info(*args, **kwargs)
+    capture(*args, **kwargs, verbosity: Logger::INFO)
+  end
+
+  def capture_with_debug(*args, **kwargs)
+    capture(*args, **kwargs, verbosity: Logger::DEBUG)
+  end
+
+  def capture_with_pretty_json(*args, **kwargs)
+    JSON.pretty_generate(JSON.parse(capture(*args, **kwargs)))
+  end
+
+  def puts_by_host(host, output, type: "App")
+    puts "#{type} Host: #{host}\n#{output}\n\n"
+  end
+
+  # Our execution pattern is for the CLI execute args lists returned
+  # from commands, but this doesn't support returning execution options
+  # from the command.
+  #
+  # Support this by using kwargs for CLI options and merging with the
+  # args-extracted options.
+  module CommandEnvMerge
+    private
+
+    # Override to merge options returned by commands in the args list with
+    # options passed by the CLI and pass them along as kwargs.
+    def command(args, options)
+      more_options, args = args.partition { |a| a.is_a? Hash }
+      more_options << options
+
+      build_command(args, **more_options.reduce(:deep_merge))
+    end
+
+    # Destructure options to pluck out env for merge
+    def build_command(args, env: nil, **options)
+      # Rely on native Ruby kwargs precedence rather than explicit Hash merges
+      SSHKit::Command.new(*args, **default_command_options, **options, env: env_for(env))
+    end
+
+    def default_command_options
+      { in: pwd_path, host: @host, user: @user, group: @group }
+    end
+
+    def env_for(env)
+      @env.to_h.merge(env.to_h)
+    end
+  end
+  prepend CommandEnvMerge
+end
+
+class SSHKit::Backend::Netssh::Configuration
+  attr_accessor :max_concurrent_starts
+end
+
+class SSHKit::Backend::Netssh
+  module LimitConcurrentStartsClass
+    attr_reader :start_semaphore
+
+    def configure(&block)
+      super &block
+      # Create this here to avoid lazy creation by multiple threads
+      if config.max_concurrent_starts
+        @start_semaphore = Concurrent::Semaphore.new(config.max_concurrent_starts)
+      end
+    end
+  end
+
+  class << self
+    prepend LimitConcurrentStartsClass
+  end
+
+  module LimitConcurrentStartsInstance
+    private
+      def with_ssh(&block)
+        host.ssh_options = self.class.config.ssh_options.merge(host.ssh_options || {})
+        self.class.pool.with(
+          method(:start_with_concurrency_limit),
+          String(host.hostname),
+          host.username,
+          host.netssh_options,
+          &block
+        )
+      end
+
+      def start_with_concurrency_limit(*args)
+        if self.class.start_semaphore
+          self.class.start_semaphore.acquire do
+            Net::SSH.start(*args)
+          end
+        else
+          Net::SSH.start(*args)
+        end
+      end
+  end
+
+  prepend LimitConcurrentStartsInstance
+end
+
+class SSHKit::Runner::Parallel
+  # SSHKit joins the threads in sequence and fails on the first error it encounters, which means that we wait threads
+  # before the first failure to complete but not for ones after.
+  #
+  # We'll patch it to wait for them all to complete, and to record all the threads that errored so we can see when a
+  # problem occurs on multiple hosts.
+  module CompleteAll
+    def execute
+      threads = hosts.map do |host|
+        Thread.new(host) do |h|
+          backend(h, &block).run
+        rescue ::StandardError => e
+          e2 = SSHKit::Runner::ExecuteError.new e
+          raise e2, "Exception while executing #{host.user ? "as #{host.user}@" : "on host "}#{host}: #{e.message}"
+        end
+      end
+
+      exceptions = []
+      threads.each do |t|
+        begin
+          t.join
+        rescue SSHKit::Runner::ExecuteError => e
+          exceptions << e
+        end
+      end
+      if exceptions.one?
+        raise exceptions.first
+      elsif exceptions.many?
+        raise exceptions.first, [ "Exceptions on #{exceptions.count} hosts:", exceptions.map(&:message) ].join("\n")
+      end
+    end
+  end
+
+  prepend CompleteAll
+end

data/lib/kamal/tags.rb

@@ -0,0 +1,40 @@
+require "time"
+
+class Kamal::Tags
+  attr_reader :config, :tags
+
+  class << self
+    def from_config(config, **extra)
+      new(**default_tags(config), **extra)
+    end
+
+    def default_tags(config)
+      { recorded_at: Time.now.utc.iso8601,
+        performer: Kamal::Git.email.presence || `whoami`.chomp,
+        destination: config.destination,
+        version: config.version,
+        service_version: service_version(config),
+        service: config.service }
+    end
+
+    def service_version(config)
+      [ config.service, config.abbreviated_version ].compact.join("@")
+    end
+  end
+
+  def initialize(**tags)
+    @tags = tags.compact
+  end
+
+  def env
+    tags.transform_keys { |detail| "KAMAL_#{detail.upcase}" }
+  end
+
+  def to_s
+    tags.values.map { |value| "[#{value}]" }.join(" ")
+  end
+
+  def except(*tags)
+    self.class.new(**self.tags.except(*tags))
+  end
+end

data/lib/kamal/utils/sensitive.rb

@@ -0,0 +1,20 @@
+require "active_support/core_ext/module/delegation"
+require "sshkit"
+
+class Kamal::Utils::Sensitive
+  # So SSHKit knows to redact these values.
+  include SSHKit::Redaction
+
+  attr_reader :unredacted, :redaction
+  delegate :to_s, to: :unredacted
+  delegate :inspect, to: :redaction
+
+  def initialize(value, redaction: "[REDACTED]")
+    @unredacted, @redaction = value, redaction
+  end
+
+  # Sensitive values won't leak into YAML output.
+  def encode_with(coder)
+    coder.represent_scalar nil, redaction
+  end
+end

data/lib/kamal/utils.rb

@@ -0,0 +1,110 @@
+require "active_support/core_ext/object/try"
+
+module Kamal::Utils
+  extend self
+
+  DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX = /\$(?!{[^\}]*\})/
+
+  # Return a list of escaped shell arguments using the same named argument against the passed attributes (hash or array).
+  def argumentize(argument, attributes, sensitive: false)
+    Array(attributes).flat_map do |key, value|
+      if value.present?
+        attr = "#{key}=#{escape_shell_value(value)}"
+        attr = self.sensitive(attr, redaction: "#{key}=[REDACTED]") if sensitive
+        [ argument, attr ]
+      elsif value == false
+        [ argument, "#{key}=false" ]
+      else
+        [ argument, key ]
+      end
+    end
+  end
+
+  # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
+  def optionize(args, with: nil)
+    options = if with
+      flatten_args(args).collect { |(key, value)| value == true ? "--#{key}" : "--#{key}#{with}#{escape_shell_value(value)}" }
+    else
+      flatten_args(args).collect { |(key, value)| [ "--#{key}", value == true ? nil : escape_shell_value(value) ] }
+    end
+
+    options.flatten.compact
+  end
+
+  # Flattens a one-to-many structure into an array of two-element arrays each containing a key-value pair
+  def flatten_args(args)
+    args.flat_map { |key, value| value.try(:map) { |entry| [ key, entry ] } || [ [ key, value ] ] }
+  end
+
+  # Marks sensitive values for redaction in logs and human-visible output.
+  # Pass `redaction:` to change the default `"[REDACTED]"` redaction, e.g.
+  # `sensitive "#{arg}=#{secret}", redaction: "#{arg}=xxxx"
+  def sensitive(...)
+    Kamal::Utils::Sensitive.new(...)
+  end
+
+  def redacted(value)
+    case
+    when value.respond_to?(:redaction)
+      value.redaction
+    when value.respond_to?(:transform_values)
+      value.transform_values { |value| redacted value }
+    when value.respond_to?(:map)
+      value.map { |element| redacted element }
+    else
+      value
+    end
+  end
+
+  # Escape a value to make it safe for shell use.
+  def escape_shell_value(value)
+    value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/) \
+      .map { |part| part.ascii_only? ? escape_ascii_shell_value(part) : part }
+      .join
+  end
+
+  def escape_ascii_shell_value(value)
+    value.to_s.dump
+      .gsub(/`/, '\\\\`')
+      .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
+  end
+
+  # Apply a list of host or role filters, including wildcard matches
+  def filter_specific_items(filters, items)
+    matches = []
+
+    Array(filters).select do |filter|
+      matches += Array(items).select do |item|
+        # Only allow * for a wildcard
+        # items are roles or hosts
+        File.fnmatch(filter, item.to_s, File::FNM_EXTGLOB)
+      end
+    end
+
+    matches.uniq
+  end
+
+  def stable_sort!(elements, &block)
+    elements.sort_by!.with_index { |element, index| [ block.call(element), index ] }
+  end
+
+  def join_commands(commands)
+    commands.map(&:strip).join(" ")
+  end
+
+  def docker_arch
+    arch = `docker info --format '{{.Architecture}}'`.strip
+    case arch
+    when /aarch64/
+      "arm64"
+    when /x86_64/
+      "amd64"
+    else
+      arch
+    end
+  end
+
+  def older_version?(version, other_version)
+    Gem::Version.new(version.delete_prefix("v")) < Gem::Version.new(other_version.delete_prefix("v"))
+  end
+end

data/lib/kamal/version.rb

@@ -0,0 +1,3 @@
+module Kamal
+  VERSION = "2.7.0"
+end

data/lib/kamal.rb

@@ -0,0 +1,14 @@
+module Kamal
+  class ConfigurationError < StandardError; end
+end
+
+require "active_support"
+require "zeitwerk"
+require "yaml"
+require "tmpdir"
+require "pathname"
+
+loader = Zeitwerk::Loader.for_gem
+loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
+loader.setup
+loader.eager_load_namespace(Kamal::Cli) # We need all commands loaded.