kitsune-kit 0.5.0 → 0.6.0

data/lib/kitsune/kit/configuration.rb

@@ -44,8 +44,27 @@ module Kitsune
         end
       end
 
+      class Compose < Data.define(:mode, :file, :allow_unsafe)
+        MODES = %w[generated overlay custom].freeze
+
+        def initialize(mode: "generated", file: nil, allow_unsafe: false, root: Dir.pwd)
+          expanded_file = file.to_s.empty? ? nil : Pathname(file.to_s).expand_path(root).to_s
+          super(mode: mode.to_s, file: expanded_file, allow_unsafe: boolean(allow_unsafe))
+        end
+
+        private
+
+        def boolean(value)
+          return value if [true, false].include?(value)
+          return true if value.to_s.casecmp("true").zero?
+          return false if value.to_s.casecmp("false").zero?
+
+          raise Errors::ConfigurationError, "expected a boolean, got #{value.inspect}"
+        end
+      end
+
       class Service < Data.define(:enabled, :mode, :host, :image, :publish, :bind, :allowed_cidrs, :port,
-                                  :password_env)
+                                  :password_env, :compose)
         def initialize(**attributes)
           super(
             enabled: boolean(attributes.fetch(:enabled)),
@@ -56,7 +75,8 @@ module Kitsune
             bind: attributes.fetch(:bind).to_s,
             allowed_cidrs: Array(attributes.fetch(:allowed_cidrs)).map(&:to_s).freeze,
             port: Integer(attributes.fetch(:port)),
-            password_env: attributes.fetch(:password_env).to_s
+            password_env: attributes.fetch(:password_env).to_s,
+            compose: attributes.fetch(:compose, Compose.new)
           )
         rescue ArgumentError, TypeError
           raise Errors::ConfigurationError, "service port must be an integer"
@@ -142,7 +162,8 @@ module Kitsune
             "bind" => "127.0.0.1",
             "allowed_cidrs" => [],
             "port" => 5432,
-            "password_env" => "POSTGRES_PASSWORD"
+            "password_env" => "POSTGRES_PASSWORD",
+            "compose" => { "mode" => "generated", "file" => nil, "allow_unsafe" => false }
           },
           "redis" => {
             "enabled" => false,
@@ -153,7 +174,8 @@ module Kitsune
             "bind" => "127.0.0.1",
             "allowed_cidrs" => [],
             "port" => 6379,
-            "password_env" => "REDIS_PASSWORD"
+            "password_env" => "REDIS_PASSWORD",
+            "compose" => { "mode" => "generated", "file" => nil, "allow_unsafe" => false }
           }
         },
         "system" => {
@@ -185,17 +207,20 @@ module Kitsune
         %w[server] => %w[name region size image ssh_key_id tags],
         %w[ssh] => %w[user port key_path allowed_cidrs],
         %w[services] => %w[postgres redis],
-        %w[services postgres] => %w[enabled mode host image publish bind allowed_cidrs port password_env],
-        %w[services redis] => %w[enabled mode host image publish bind allowed_cidrs port password_env],
+        %w[services postgres] => %w[enabled mode host image publish bind allowed_cidrs port password_env compose],
+        %w[services redis] => %w[enabled mode host image publish bind allowed_cidrs port password_env compose],
+        %w[services postgres compose] => %w[mode file allow_unsafe],
+        %w[services redis compose] => %w[mode file allow_unsafe],
         %w[system] => %w[swap_size_gb swap_swappiness unattended_upgrades metrics metrics_installer_sha256],
         %w[dns] => %w[domains ttl]
       }.freeze
 
       class Loader
-        def initialize(root: Dir.pwd, env: ENV, config_path: nil)
+        def initialize(root: Dir.pwd, env: ENV, config_path: nil, validate_secrets: true)
           @root = Pathname(root).expand_path
           @env = env
           @config_path = config_path ? Pathname(config_path).expand_path : @root.join(".kitsune/config.yml")
+          @validate_secrets = validate_secrets
         end
 
         def load(environment: nil, overrides: {})
@@ -260,13 +285,13 @@ module Kitsune
             server: Server.new(**symbolize(data.fetch("server"))),
             ssh: Ssh.new(**symbolize(data.fetch("ssh"))),
             services: Services.new(
-              postgres: Service.new(**symbolize(data.dig("services", "postgres"))),
-              redis: Service.new(**symbolize(data.dig("services", "redis")))
+              postgres: build_service(data.dig("services", "postgres")),
+              redis: build_service(data.dig("services", "redis"))
             ),
             system: System.new(**symbolize(data.fetch("system"))),
             dns: Dns.new(**symbolize(data.fetch("dns")))
           )
-          Validator.new(config, env: @env).validate!
+          Validator.new(config, env: @env, root: @root, validate_secrets: @validate_secrets).validate!
           config
         rescue KeyError => e
           raise Errors::ConfigurationError.new("missing configuration value: #{e.key}",
@@ -293,6 +318,12 @@ module Kitsune
           )
         end
 
+        def build_service(data)
+          attributes = symbolize(data)
+          compose = Compose.new(**symbolize(data.fetch("compose")), root: @root)
+          Service.new(**attributes, compose: compose)
+        end
+
         def validate_schema_version!(version)
           return if version == SCHEMA_VERSION
 
@@ -335,9 +366,11 @@ module Kitsune
         IMAGE = %r{\A[a-z0-9]+(?:[._/-][a-z0-9]+)*(?::[a-zA-Z0-9][a-zA-Z0-9._-]*)?(?:@sha256:[a-f0-9]{64})?\z}
         INSECURE_SECRETS = %w[password postgres redis changeme change-me secret admin].freeze
 
-        def initialize(config, env: ENV)
+        def initialize(config, env: ENV, root: Dir.pwd, validate_secrets: true)
           @config = config
           @env = env
+          @root = Pathname(root).expand_path
+          @validate_secrets = validate_secrets
           @errors = []
         end
 
@@ -410,7 +443,58 @@ module Kitsune
           else
             validate_managed_service(name, service)
           end
-          validate_service_secret(name, service) if service.enabled
+          validate_compose(name, service)
+          validate_service_secret(name, service) if service.enabled && @validate_secrets
+        end
+
+        def validate_compose(name, service)
+          compose = service.compose
+          return invalid_compose_mode(name) unless Compose::MODES.include?(compose.mode)
+
+          validate_compose_ownership(name, service)
+          return validate_generated_compose(name, compose) if compose.mode == "generated"
+          return missing_compose_file(name, compose) unless compose.file
+
+          validate_compose_file(name, Pathname(compose.file))
+        rescue SystemCallError => e
+          @errors << "services.#{name}.compose.file cannot be inspected: #{e.class}"
+        end
+
+        def invalid_compose_mode(name)
+          @errors << "services.#{name}.compose.mode must be generated, overlay or custom"
+        end
+
+        def validate_compose_ownership(name, service)
+          return unless service.mode == "external" && service.compose.mode != "generated"
+
+          @errors << "services.#{name}.compose is only available in managed mode"
+        end
+
+        def validate_generated_compose(name, compose)
+          @errors << "services.#{name}.compose.file must be empty in generated mode" if compose.file
+          return unless compose.allow_unsafe
+
+          @errors << "services.#{name}.compose.allow_unsafe must be false in generated mode"
+        end
+
+        def missing_compose_file(name, compose)
+          @errors << "services.#{name}.compose.file is required in #{compose.mode} mode"
+        end
+
+        def validate_compose_file(name, path)
+          root_prefix = "#{@root}#{File::SEPARATOR}"
+          return @errors << "services.#{name}.compose.file must stay inside the project root" unless
+            path.to_s.start_with?(root_prefix)
+
+          if path.symlink?
+            @errors << "services.#{name}.compose.file must not be a symbolic link"
+          elsif !path.file?
+            @errors << "services.#{name}.compose.file does not exist or is not a regular file: #{path}"
+          elsif !path.realpath.to_s.start_with?("#{@root.realpath}#{File::SEPARATOR}")
+            @errors << "services.#{name}.compose.file must not resolve outside the project root"
+          elsif path.size > 262_144
+            @errors << "services.#{name}.compose.file must not exceed 256 KiB"
+          end
         end
 
         def validate_managed_service(name, service)

data/lib/kitsune/kit/operations/ensure_service.rb

@@ -12,7 +12,8 @@ require_relative "service_state"
 module Kitsune
   module Kit
     module Operations
-      class EnsureService
+      # Service lifecycle coordination intentionally lives together so apply and rollback share one transaction model.
+      class EnsureService # rubocop:disable Metrics/ClassLength
         TYPES = %w[postgres redis].freeze
         MARKER_DIRECTORY = "/var/lib/kitsune/state"
 
@@ -62,7 +63,11 @@ module Kitsune
           transport.execute("mkdir", arguments: ["-p", service_directory], timeout: 10).then do |result|
             raise_remote!(result, "create service directory")
           end
-          transport.upload(content: compose_content, remote_path: compose_path, mode: "0600")
+          @compose.documents.each do |document|
+            transport.upload(
+              content: document.content, remote_path: files.compose_path(document.filename), mode: "0600"
+            )
+          end
           transport.upload(content: env_content(password), remote_path: env_path, mode: "0600")
           validate_compose
           start_service
@@ -79,7 +84,7 @@ module Kitsune
         def remove
           return false unless managed?
 
-          compose("down", "--remove-orphans")
+          compose("down", "--remove-orphans", filenames: installed_compose_files)
           firewall.remove
           remove_marker
           @managed_state.record_remove
@@ -97,7 +102,8 @@ module Kitsune
           remove
           if files.restore_previous(previous, marker_path: marker_path)
             firewall.restore(previous["previous_state"] || {})
-            start_service if previous.dig("previous_state", "installed") != false
+            start_service(filenames: previous.dig("previous_state", "compose_files")) if
+              previous.dig("previous_state", "installed") != false
           end
           @managed_state.record_rollback(previous)
           true
@@ -107,7 +113,7 @@ module Kitsune
           previous = managed_state
           return false unless previous
 
-          compose("down", "--volumes", "--remove-orphans")
+          compose("down", "--volumes", "--remove-orphans", filenames: installed_compose_files)
           files.destroy(previous, marker_path: marker_path)
           @managed_state.delete("destroy_data")
           true
@@ -120,13 +126,12 @@ module Kitsune
         def transport = @transport ||= transport_factory.deploy
         def project_name = "kitsune-#{config.environment}-#{@type}"
         def service_directory = files.directory
-        def compose_path = files.compose_path
         def env_path = files.env_path
         def marker_path = "#{MARKER_DIRECTORY}/service-#{@type}.sha256"
 
         def fingerprint
           @fingerprint ||= Digest::SHA256.hexdigest(
-            [compose_content, service.password_env, secret_digest, service.publish, service.bind,
+            [@compose.fingerprint, service.password_env, secret_digest, service.publish, service.bind,
              *service.allowed_cidrs].join("\0")
           )
         end
@@ -147,21 +152,24 @@ module Kitsune
           value
         end
 
-        def compose_content = @compose.content
         def env_content(password) = @compose.env_content(password)
 
         def validate_compose = compose("config", "--quiet")
-        def start_service = compose("up", "--detach", "--wait", "--wait-timeout", "120")
+
+        def start_service(filenames: nil)
+          compose("up", "--detach", "--wait", "--wait-timeout", "120", filenames: filenames)
+        end
 
         def verify_service
           result = compose("ps", "--status", "running", "--quiet")
           raise Errors::VerificationError, "#{@type} did not reach running state" if result.stdout.strip.empty?
         end
 
-        def compose(*arguments)
+        def compose(*arguments, filenames: nil)
           result = transport.execute(
             "docker",
-            arguments: ["compose", "--project-directory", service_directory, "--file", compose_path, *arguments],
+            arguments: ["compose", "--project-directory", service_directory,
+                        *compose_file_arguments(filenames), *arguments],
             timeout: 180
           )
           raise_remote!(result, "docker compose #{arguments.first}")
@@ -176,7 +184,7 @@ module Kitsune
           firewall.restore_after_failure(previous_state) if @firewall_changed
           safe_compose_down
           files.restore_after_failure(@backed_up_files)
-          start_service unless @backed_up_files.empty?
+          start_service(filenames: previous_state&.fetch("compose_files", nil)) unless @backed_up_files.empty?
         end
 
         def recover_failed_apply(previous_state, original_error)
@@ -194,7 +202,7 @@ module Kitsune
         def safe_compose_down
           transport.execute(
             "docker",
-            arguments: ["compose", "--project-directory", service_directory, "--file", compose_path,
+            arguments: ["compose", "--project-directory", service_directory, *compose_file_arguments,
                         "down", "--remove-orphans"],
             timeout: 180
           )
@@ -219,6 +227,8 @@ module Kitsune
           {
             "managed" => true, "installed" => true, "fingerprint" => fingerprint, "project" => project_name,
             "directory" => service_directory, "published" => service.publish, "port" => service.port,
+            "compose_mode" => @compose.mode, "compose_files" => @compose.filenames,
+            "compose_fingerprint" => @compose.fingerprint,
             "firewall_rules_added" => firewall.owned_rules, "firewall_drop_added" => firewall.drop_owned,
             "volume" => "#{project_name}_data",
             "previous_fingerprint" => change.details[:actual_fingerprint] || change.details["actual_fingerprint"],
@@ -229,11 +239,12 @@ module Kitsune
 
         def managed? = @managed_state.managed?
         def managed_state = @managed_state.current
+        def installed_compose_files = managed_state&.fetch("compose_files", nil)
 
         def files
           @files ||= ServiceFiles.new(
             config: config, type: @type, transport: transport, state_store: state_store,
-            resource: resource, fingerprint: -> { fingerprint }
+            resource: resource, fingerprint: -> { fingerprint }, compose_filenames: -> { @compose.filenames }
           )
         end
 
@@ -246,7 +257,8 @@ module Kitsune
 
         def backup_operation
           @backup_operation ||= ServiceBackup.new(
-            config: config, type: @type, transport: transport, compose: method(:compose), clock: @clock
+            config: config, type: @type, transport: transport,
+            compose: ->(*arguments) { compose(*arguments, filenames: installed_compose_files) }, clock: @clock
           )
         end
 
@@ -274,10 +286,15 @@ module Kitsune
             image: service.image, published: service.publish,
             bind: service.publish ? service.bind : nil, port: service.publish ? service.port : nil,
             fingerprint: fingerprint,
-            actual_fingerprint: actual.empty? ? nil : actual
+            actual_fingerprint: actual.empty? ? nil : actual,
+            compose: @compose.metadata
           }
         end
 
+        def compose_file_arguments(filenames = nil)
+          Array(filenames || @compose.filenames).flat_map { |filename| ["--file", files.compose_path(filename)] }
+        end
+
         def error_name(error) = error.respond_to?(:code) ? error.code : error.class.name
       end
     end

data/lib/kitsune/kit/operations/service_files.rb

@@ -6,23 +6,25 @@ module Kitsune
   module Kit
     module Operations
       class ServiceFiles
-        def initialize(config:, type:, transport:, state_store:, resource:, fingerprint:)
+        def initialize(config:, type:, transport:, state_store:, resource:, fingerprint:, compose_filenames:)
           @config = config
           @type = type
           @transport = transport
           @state_store = state_store
           @resource = resource
           @fingerprint = fingerprint
+          @compose_filenames = compose_filenames
         end
 
         def directory = "/home/#{config.ssh.user}/.local/share/kitsune/services/#{type}"
-        def compose_path = "#{directory}/compose.yml"
+        def compose_path(filename = "compose.yml") = "#{directory}/#{filename}"
+        def compose_paths = compose_filenames.call.map { |filename| compose_path(filename) }
         def env_path = "#{directory}/.env"
         def backup_directory = "/var/lib/kitsune/backups/service-#{type}-#{fingerprint.call[0, 12]}"
 
         def ensure_managed_or_absent!(actual_fingerprint)
           return if managed?
-          return if actual_fingerprint.empty? && !exists?(compose_path) && !exists?(env_path)
+          return if actual_fingerprint.empty? && known_paths.none? { |path| exists?(path) }
 
           raise Errors::UnsafeOperationError.new(
             "refusing to replace unmanaged #{type} service files",
@@ -39,17 +41,14 @@ module Kitsune
             timeout: 20
           )
           raise_remote!(result, "create service backup directory")
-          [compose_path, env_path].select { |path| exists?(path) }.tap do |paths|
+          known_paths.select { |path| exists?(path) }.tap do |paths|
             paths.each { |path| copy(path, backup_directory) }
           end
         end
 
         def restore_after_failure(backed_up_files)
-          if backed_up_files.empty?
-            transport.execute("rm", arguments: ["-f", compose_path, env_path], timeout: 20)
-          else
-            backed_up_files.each { |path| copy("#{backup_directory}/#{File.basename(path)}", File.dirname(path)) }
-          end
+          transport.execute("rm", arguments: ["-f", *known_paths], timeout: 20)
+          backed_up_files.each { |path| copy("#{backup_directory}/#{File.basename(path)}", File.dirname(path)) }
         end
 
         def restore_previous(previous, marker_path:)
@@ -75,7 +74,12 @@ module Kitsune
 
         private
 
-        attr_reader :config, :type, :transport, :state_store, :resource, :fingerprint
+        attr_reader :config, :type, :transport, :state_store, :resource, :fingerprint, :compose_filenames
+
+        def known_paths
+          previous = state_store.read(config.environment).dig("resources", resource, "compose_files") || []
+          (compose_paths + previous.map { |filename| compose_path(filename) } + [env_path]).uniq
+        end
 
         def managed? = !!state_store.read(config.environment).dig("resources", resource, "managed")
         def exists?(path) = transport.execute("test", arguments: ["-e", path], timeout: 10).success?

data/lib/kitsune/kit/scripts/docker.sh

@@ -9,6 +9,11 @@ repository="/etc/apt/sources.list.d/docker.sources"
 packages=(docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin)
 prerequisites=(ca-certificates curl)
 conflicting_packages=(docker.io docker-compose docker-compose-v2 podman-docker containerd runc)
+apt_options=(-o DPkg::Lock::Timeout=120 -o Acquire::Retries=3)
+
+apt_get() {
+  DEBIAN_FRONTEND=noninteractive apt-get "${apt_options[@]}" "$@"
+}
 
 refuse_conflicting_packages() {
   local installed=()
@@ -42,8 +47,8 @@ apply() {
   refuse_conflicting_packages
   install -d -m 0700 "${backup_dir}"
   snapshot
-  apt-get update
-  DEBIAN_FRONTEND=noninteractive apt-get install --yes "${prerequisites[@]}"
+  apt_get update
+  apt_get install --yes "${prerequisites[@]}"
   install -d -m 0755 /etc/apt/keyrings
   if [[ ! -f "${keyring}" ]]; then
     temporary_key="$(mktemp)"
@@ -76,8 +81,8 @@ EOF
     trap - EXIT
     touch "${backup_dir}/repository-created"
   fi
-  apt-get update
-  DEBIAN_FRONTEND=noninteractive apt-get install --yes "${packages[@]}"
+  apt_get update
+  apt_get install --yes "${packages[@]}"
   systemctl enable --now docker
   if ! id -nG "${user}" | tr ' ' '\n' | grep -Fxq docker; then
     usermod -aG docker "${user}"
@@ -106,12 +111,12 @@ rollback() {
   fi
   for package in "${packages[@]}"; do
     if ! grep -Fxq "${package}" "${backup_dir}/packages-before" 2>/dev/null; then
-      apt-get remove --yes "${package}" || true
+      apt_get remove --yes "${package}" || true
     fi
   done
   for package in "${prerequisites[@]}"; do
     if ! grep -Fxq "${package}" "${backup_dir}/packages-before" 2>/dev/null; then
-      apt-get remove --yes "${package}" || true
+      apt_get remove --yes "${package}" || true
     fi
   done
   if [[ -f "${backup_dir}/enabled-before" ]]; then

data/lib/kitsune/kit/scripts/firewall.sh

@@ -8,6 +8,11 @@ allowed_cidrs=("$@")
 backup_dir="/var/lib/kitsune/backups/firewall"
 added_file="${backup_dir}/rules-added"
 pending_file="${backup_dir}/rules-pending"
+apt_options=(-o DPkg::Lock::Timeout=120 -o Acquire::Retries=3)
+
+apt_get() {
+  DEBIAN_FRONTEND=noninteractive apt-get "${apt_options[@]}" "$@"
+}
 
 validate() {
   [[ "${ssh_port}" =~ ^[0-9]+$ ]] && (( ssh_port >= 1 && ssh_port <= 65535 ))
@@ -127,8 +132,8 @@ apply() {
     touch "${backup_dir}/snapshot-complete"
   fi
   if ! dpkg-query -W -f='${Status}' ufw 2>/dev/null | grep -q "ok installed"; then
-    apt-get update
-    DEBIAN_FRONTEND=noninteractive apt-get install --yes ufw
+    apt_get update
+    apt_get install --yes ufw
     touch "${backup_dir}/package-created"
   fi
 
@@ -190,7 +195,7 @@ rollback() {
     ufw --force disable || true
   fi
   if [[ -f "${backup_dir}/package-created" ]]; then
-    apt-get remove --yes ufw || true
+    apt_get remove --yes ufw || true
   fi
   rm -rf "${backup_dir}"
 }

data/lib/kitsune/kit/scripts/metrics.sh

@@ -4,6 +4,11 @@ set -Eeuo pipefail
 action="${1:?action is required}"
 expected_sha256="${2:?installer SHA-256 is required}"
 backup_dir="/var/lib/kitsune/backups/metrics"
+apt_options=(-o DPkg::Lock::Timeout=120 -o Acquire::Retries=3)
+
+apt_get() {
+  DEBIAN_FRONTEND=noninteractive apt-get "${apt_options[@]}" "$@"
+}
 
 validate() {
   [[ "${expected_sha256}" =~ ^[a-f0-9]{64}$ ]] || {
@@ -41,7 +46,7 @@ verify() {
 rollback() {
   if [[ -f "${backup_dir}/package-created" ]]; then
     systemctl disable --now do-agent || true
-    apt-get remove --purge --yes do-agent || true
+    apt_get remove --purge --yes do-agent || true
   fi
   rm -rf "${backup_dir}"
 }

data/lib/kitsune/kit/scripts/unattended.sh

@@ -6,6 +6,11 @@ config_file="/etc/apt/apt.conf.d/20auto-upgrades"
 backup_dir="/var/lib/kitsune/backups/unattended"
 backup_file="${backup_dir}/20auto-upgrades.before"
 packages=(unattended-upgrades apt-listchanges)
+apt_options=(-o DPkg::Lock::Timeout=120 -o Acquire::Retries=3)
+
+apt_get() {
+  DEBIAN_FRONTEND=noninteractive apt-get "${apt_options[@]}" "$@"
+}
 
 snapshot() {
   [[ -f "${backup_dir}/snapshot-complete" ]] && return
@@ -28,8 +33,8 @@ snapshot() {
 apply() {
   install -d -m 0700 "${backup_dir}"
   snapshot
-  apt-get update
-  DEBIAN_FRONTEND=noninteractive apt-get install --yes "${packages[@]}"
+  apt_get update
+  apt_get install --yes "${packages[@]}"
   cat > "${config_file}" <<'EOF'
 APT::Periodic::Update-Package-Lists "1";
 APT::Periodic::Download-Upgradeable-Packages "1";
@@ -67,7 +72,7 @@ rollback() {
   restore_service_state
   for package in "${packages[@]}"; do
     if ! grep -Fxq "${package}" "${backup_dir}/packages-before" 2>/dev/null; then
-      apt-get remove --yes "${package}" || true
+      apt_get remove --yes "${package}" || true
     fi
   done
   rm -rf "${backup_dir}"