dash 4.1.2 → 4.2.0

data/lib/dash/configuration/role/scale.rb

@@ -0,0 +1,199 @@
+# How many HOSTS a role may run on: the baseline listed under `hosts`, plus pool members the
+# autoscale provider holds (see Dash::Autoscale::Pool). `min` and `max` count hosts, the
+# baseline included; `replicas` counts containers per host - the two never mix.
+#
+# The controller's keys (`schedule`, `warmup`, `cooldown`, `step`) count containers, the
+# unit of `dash scale set`; see Dash::Autoscale::Policy. `signal`, `up`, `down` and
+# `hold_when` are its reactive rules: the band a signal is held in, and the guards that veto
+# a scale-out it asks for.
+class Dash::Configuration::Role::Scale
+  MEMBERS = %w[ power create ]
+  ADDRESSES = %w[ private public utility ]
+  WINDOW_KEYS = %w[ cron for min ]
+  DEFAULT_BOOT_TIMEOUT = 300
+  DEFAULT_COOLDOWN_UP = 60
+  DEFAULT_COOLDOWN_DOWN = 600
+  MAX_WINDOW = 7 * 86_400
+  MAX_BREACH = 86_400
+  DEFAULT_UP_FOR = 0
+  DEFAULT_DOWN_FOR = 600
+
+  attr_reader :min, :max, :template, :address, :boot_timeout, :schedule, :warmup, :cooldown_up, :cooldown_down, :step, :replicas, :context,
+    :signal, :up_above, :up_for, :down_below, :down_for, :guards
+
+  def initialize(scale_config:, baseline:, replicas:, context:)
+    @context = context
+    @replicas = replicas
+    @min = scale_config.fetch("min", baseline)
+    @max = scale_config["max"]
+    @members = scale_config.fetch("members", "power").to_s
+    @template = scale_config["template"]
+    @address = scale_config.fetch("address", "private").to_s
+    @boot_timeout = scale_config.fetch("boot_timeout", DEFAULT_BOOT_TIMEOUT)
+    @warmup = scale_config.fetch("warmup", boot_timeout)
+    @cooldown_up = scale_config.dig("cooldown", "up") || DEFAULT_COOLDOWN_UP
+    @cooldown_down = scale_config.dig("cooldown", "down") || DEFAULT_COOLDOWN_DOWN
+    @step = scale_config.fetch("step", replicas.max)
+
+    validate!(baseline)
+    @schedule = Array(scale_config["schedule"]).each_with_index.map { |window, index| parse_window(window, index) }
+    parse_reactive(scale_config)
+  end
+
+  def members
+    @members.to_sym
+  end
+
+  def power?
+    members == :power
+  end
+
+  def create?
+    members == :create
+  end
+
+  def signal?
+    !signal.nil?
+  end
+
+  # Whether `dash autoscale run` owns the role's count: a schedule, a signal, or both. A
+  # scaled role with neither stays manual.
+  def controlled?
+    schedule.any? || signal?
+  end
+
+  # The role's bounds in containers: every host it must keep at replicas min, every host
+  # it may have at replicas max.
+  def min_count
+    min * replicas.min
+  end
+
+  def max_count
+    max * replicas.max
+  end
+
+  # Whether `dash scale set` can place `count` containers: on as few hosts as hold them,
+  # never fewer than scale min, each within the replicas bounds (Dash::Cli::Scale::HostPlan).
+  def splittable?(count)
+    hosts = [ (count.to_f / replicas.max).ceil, min ].max
+    hosts <= max && hosts * replicas.min <= count
+  end
+
+  private
+    def validate!(baseline)
+      error "max", "is required" if max.nil?
+      error "members", "must be #{MEMBERS.join(" or ")}, not #{@members}" unless MEMBERS.include?(@members)
+      error "address", "must be #{ADDRESSES.to_sentence(two_words_connector: " or ", last_word_connector: " or ")}, not #{address}" unless ADDRESSES.include?(address)
+      error nil, "min (#{min}) cannot be less than the #{baseline} hosts listed under hosts" if min < baseline
+      error nil, "min (#{min}) cannot be greater than max (#{max})" if min > max
+      error "boot_timeout", "must be at least 1 second" unless boot_timeout.positive?
+      error "warmup", "must be at least 0" if warmup.negative?
+      error "cooldown/up", "must be at least 0" if cooldown_up.negative?
+      error "cooldown/down", "must be at least 0" if cooldown_down.negative?
+      error "step", "must be at least 1" unless step.positive?
+      validate_template!
+    end
+
+    def validate_template!
+      if create?
+        error "template", "is required with members: create" if template.blank?
+        error "template/network", "is required when address is private" if address == "private" && template["network"].blank?
+      elsif template.present?
+        error "template", "is only used with members: create"
+      end
+    end
+
+    def parse_window(window, index)
+      key = "schedule/#{index}"
+      if (unknown = (window.keys.map(&:to_s) - WINDOW_KEYS).first)
+        error key, "unknown key: #{unknown}"
+      end
+      WINDOW_KEYS.each { |required| error "#{key}/#{required}", "is required" if window[required].nil? }
+
+      Window.new(cron: window_cron(window["cron"], key), duration: window_duration(window["for"], key), min: window_min(window["min"], key))
+    end
+
+    def window_cron(cron, key)
+      Dash::Autoscale::Cron.parse(cron)
+    rescue Dash::Autoscale::Cron::Error => e
+      error "#{key}/cron", e.message
+    end
+
+    def window_duration(value, key)
+      duration = begin
+        Dash::Autoscale::Duration.parse(value)
+      rescue ArgumentError => e
+        error "#{key}/for", e.message
+      end
+
+      error "#{key}/for", "must be 1 second to 7 days, not #{value}" unless duration.between?(1, MAX_WINDOW)
+      duration
+    end
+
+    # Below min_count a window changes nothing, so only a min above it must be one
+    # `dash scale set` can place.
+    def window_min(value, key)
+      unless value.is_a?(Integer) && value.between?(1, max_count)
+        error "#{key}/min", "must be 1 to #{max_count} (scale max #{max} × replicas max #{replicas.max}), not #{value}"
+      end
+
+      if value > min_count && !splittable?(value)
+        error "#{key}/min", "#{value} containers cannot be split over the role's hosts " \
+          "(replicas min #{replicas.min}, max #{replicas.max} per host, #{min} to #{max} hosts)"
+      end
+
+      value
+    end
+
+    def parse_reactive(scale_config)
+      if scale_config.key?("signal")
+        @signal = Signal.new(signal_config: scale_config["signal"], context: [ context, "signal" ].join("/"))
+        %w[ up down ].each { |key| error key, "is required with a signal" unless scale_config.key?(key) }
+      else
+        %w[ up down hold_when ].each { |key| error key, "needs a signal" if scale_config.key?(key) }
+        @guards = []
+        return
+      end
+
+      @up_above, @up_for = parse_band(scale_config["up"], "up", "above", DEFAULT_UP_FOR)
+      @down_below, @down_for = parse_band(scale_config["down"], "down", "below", DEFAULT_DOWN_FOR)
+      error nil, "up/above (#{up_above}) must be greater than down/below (#{down_below})" unless up_above > down_below
+      @guards = parse_guards(scale_config.fetch("hold_when", []))
+    end
+
+    # `up: { above:, for: }` or `down: { below:, for: }`: the bound, and how long the signal
+    # stays past it before the controller acts.
+    def parse_band(band, key, bound_key, default_for)
+      error key, "should be a hash" unless band.is_a?(Hash)
+      if (unknown = (band.keys.map(&:to_s) - [ bound_key, "for" ]).first)
+        error key, "unknown key: #{unknown}"
+      end
+
+      bound = band[bound_key]
+      error "#{key}/#{bound_key}", "is required" if bound.nil?
+      error "#{key}/#{bound_key}", "should be a finite number" unless bound.is_a?(Numeric) && bound.real? && bound.finite?
+
+      [ bound, breach_duration(band.fetch("for", default_for), "#{key}/for") ]
+    end
+
+    def breach_duration(value, key)
+      duration = begin
+        Dash::Autoscale::Duration.parse(value)
+      rescue ArgumentError => e
+        error key, e.message
+      end
+
+      error key, "must be 0 seconds to 1 day, not #{value}" unless duration.between?(0, MAX_BREACH)
+      duration
+    end
+
+    def parse_guards(guards)
+      error "hold_when", "should be an array" unless guards.is_a?(Array)
+
+      guards.each_with_index.map { |guard, index| Guard.new(guard_config: guard, context: [ context, "hold_when/#{index}" ].join("/")) }
+    end
+
+    def error(key, message)
+      raise Dash::ConfigurationError, "#{[ context, key ].compact.join("/")}: #{message}"
+    end
+end

data/lib/dash/configuration/role.rb

@@ -3,7 +3,10 @@ class Dash::Configuration::Role
 
   delegate :argumentize, :optionize, to: Dash::Utils
 
-  attr_reader :name, :config, :specialized_env, :specialized_logging, :specialized_proxy, :healthcheck
+  attr_reader :name, :config, :specialized_env, :specialized_logging, :specialized_proxy, :healthcheck, :replicas, :drain, :scale
+
+  delegate :numbers, to: :replicas, prefix: :replica
+  delegate :signal, :wait, to: :drain, prefix: true
 
   alias to_s name
 
@@ -31,19 +34,73 @@ class Dash::Configuration::Role
         context: "servers/#{name}/healthcheck"
     end
 
+    @replicas = Dash::Configuration::Role::Replicas.new \
+      replicas_config: specializations["replicas"],
+      context: "servers/#{name}/replicas"
+
+    @drain = Dash::Configuration::Role::Drain.new \
+      drain_config: specializations["drain"],
+      context: "servers/#{name}/drain"
+
+    if (scale_config = specializations["scale"])
+      @scale = Dash::Configuration::Role::Scale.new \
+        scale_config: scale_config,
+        baseline: baseline_hosts.size,
+        replicas: replicas,
+        context: "servers/#{name}/scale"
+    end
+
     initialize_specialized_proxy
+
+    if running_proxy? && drain.configured?
+      raise Dash::ConfigurationError, "servers/#{name}/drain: a role behind dash-proxy is drained by the proxy, remove drain"
+    end
   end
 
   def primary_host
     hosts.first
   end
 
+  # Baseline hosts first, then the pool members the provider reports started. Every command
+  # that reads a role's hosts sees the members; nothing else has to know about them.
   def hosts
+    scaled? ? (baseline_hosts + active_members.map(&:host)).uniq : baseline_hosts
+  end
+
+  # The hosts deploy.yml lists for the role. Configuration-time checks read these and never
+  # #hosts, which for a scaled role asks the autoscale provider.
+  def baseline_hosts
     tagged_hosts.keys
   end
 
+  def scaled?
+    !scale.nil?
+  end
+
+  # Whether `dash autoscale run` owns the role's count (a scale schedule or signal).
+  def controlled?
+    scaled? && scale.controlled?
+  end
+
+  # Every pool member of the role, whatever its state.
+  def members
+    scaled? ? config.pool.members_for(self) : []
+  end
+
+  def active_members
+    members.select(&:started?)
+  end
+
+  # `host` may be an SSHKit::Host - `on` hands those to its blocks.
+  def member_host?(host)
+    host = host.to_s
+    !baseline_hosts.include?(host) && active_members.any? { |member| member.host == host }
+  end
+
+  # A member carries no tags: deploy.yml does not list it.
   def env_tags(host)
-    tagged_hosts.fetch(host).collect { |tag| config.env_tag(tag) }.compact
+    tags = tagged_hosts.fetch(host) { member_host?(host) ? [] : raise(KeyError, "#{host} is not a host of role #{name}") }
+    tags.collect { |tag| config.env_tag(tag) }.compact
   end
 
   # The role's own hosts carrying `tag` in deploy.yml. Accessory `tag:`/`tags:` resolution
@@ -217,11 +274,66 @@ class Dash::Configuration::Role
 
 
   def container_name(version = nil)
-    [ container_prefix, version || config.version ].compact.join("-")
+    replica_name(1, version)
   end
 
+  # Also the dash-proxy service name, which stays one per role whatever the replica count.
   def container_prefix
-    [ config.service, name, config.destination ].compact.join("-")
+    replica_prefix(1)
+  end
+
+  # Replica 1 is the container the role always had, so its name never changed; replica n
+  # puts the slot into the role segment, where a free-form version cannot reach it.
+  def replica_prefix(replica)
+    [ config.service, replica == 1 ? name : "#{name}.#{replica}", config.destination ].compact.join("-")
+  end
+
+  # Whether docker lookups for a slot need its name as well as the role labels: a scalable
+  # role's slot 1 would otherwise see slot 2's containers. Slot 2 and up always do, even
+  # after `replicas` was lowered to 1, so a deploy can still find and stop them.
+  def replica_scoped?(replica)
+    replica > 1 || replicas.scalable?
+  end
+
+  # The `--filter name=` value for a slot's containers. Docker matches it as a regular
+  # expression, so a `.` in the slot or the destination has to be literal.
+  def replica_name_filter(replica)
+    "'name=^#{replica_name_pattern(replica)}-'"
+  end
+
+  # The slot's prefix as a docker name regex: slot n's `.` has to be literal, or
+  # `^app-web.2-123$` would also match slot 1's container of a version named `2-123`.
+  def replica_name_pattern(replica)
+    replica_prefix(replica).gsub(/[.^$*+?()\[\]{}|\\]/) { |char| "\\#{char}" }
+  end
+
+  def replica_name(replica, version = nil)
+    [ replica_prefix(replica), version || config.version ].compact.join("-")
+  end
+
+  # The slot a container of this role is, read from its name - nil when the name is not
+  # one of this role's. The `role` label already excludes a sibling role whose name extends
+  # this one, so callers only ever hand this names of this role's containers.
+  def replica_from_name(container_name)
+    if container_name.start_with?("#{replica_prefix(1)}-")
+      1
+    elsif (match = replica_name_regexp.match(container_name))
+      match[1].to_i
+    end
+  end
+
+  def version_from_name(container_name)
+    if (replica = replica_from_name(container_name))
+      container_name.delete_prefix("#{replica_prefix(replica)}-")
+    end
+  end
+
+  def docker_option_keys
+    docker_options.keys.map(&:to_s)
+  end
+
+  def docker_option_values(*keys)
+    docker_options.select { |key, _| keys.include?(key.to_s) }.values.flatten
   end
 
 
@@ -254,8 +366,8 @@ class Dash::Configuration::Role
 
   def ensure_one_host_for_ssl
     # Skip SSL validation when a loadbalancer is present or custom certificates are provided
-    if running_proxy? && proxy.ssl? && hosts.size > 1 && !proxy.loadbalancer.present? && !proxy.custom_ssl_certificate?
-      raise Dash::ConfigurationError, "SSL is only supported on a single server unless you provide custom certificates or configure a loadbalancer, found #{hosts.size} servers for role #{name}"
+    if running_proxy? && proxy.ssl? && baseline_hosts.size > 1 && !proxy.loadbalancer.present? && !proxy.custom_ssl_certificate?
+      raise Dash::ConfigurationError, "SSL is only supported on a single server unless you provide custom certificates or configure a loadbalancer, found #{baseline_hosts.size} servers for role #{name}"
     end
   end
 
@@ -305,6 +417,11 @@ class Dash::Configuration::Role
       end
     end
 
+    def replica_name_regexp
+      destination = "-#{Regexp.escape(config.destination)}" if config.destination
+      /\A#{Regexp.escape(config.service)}-#{Regexp.escape(name)}\.(\d+)#{destination}-/
+    end
+
     def default_labels
       { "service" => config.service, "role" => name, "destination" => config.destination }
     end

data/lib/dash/configuration/validator/autoscale.rb

@@ -0,0 +1,126 @@
+class Dash::Configuration::Validator::Autoscale < Dash::Configuration::Validator
+  PROVIDERS = %w[ upcloud exec ]
+  CREDENTIALS = %w[ username password ]
+  SCRIPTS = %w[ members start stop create destroy ]
+  REQUIRED_SCRIPTS = %w[ members start stop ]
+
+  MINIMUM_INTERVAL = 5
+
+  # What a signal or guard reads through, and the URL schemes each one takes.
+  SOURCES = {
+    "redis" => %w[ redis rediss ],
+    "postgres" => %w[ postgres postgresql ],
+    "prometheus" => %w[ http https ]
+  }.freeze
+
+  def validate!
+    validate_type! config, Hash
+    check_unknown_keys! config, example
+    validate_controller!
+    SOURCES.each_key { |source| validate_source!(source, config[source]) if config.key?(source) }
+
+    with_context("provider") do
+      provider = config["provider"]
+      error "is required" if provider.nil?
+      validate_type! provider, Hash
+      # Against the constants, not the example: the docs show each provider on its own.
+      check_unknown_keys! provider, PROVIDERS.index_with(nil)
+      error "set exactly one of #{PROVIDERS.join(" or ")}" unless provider.size == 1
+
+      validate_upcloud!(provider["upcloud"]) if provider.key?("upcloud")
+      validate_exec!(provider["exec"]) if provider.key?("exec")
+    end
+  end
+
+  private
+    def validate_controller!
+      with_context("controller") do
+        host = config["controller"]
+        error "should be a host name" unless host.nil? || (host.is_a?(String) && host.present? && host == host.strip)
+      end
+      with_context("interval") { validate_at_least! config["interval"], MINIMUM_INTERVAL, " seconds" }
+      with_context("lock_wait_timeout") { validate_at_least! config["lock_wait_timeout"], 0 }
+
+      with_context("timezone") do
+        timezone = config["timezone"]
+        next if timezone.nil?
+
+        error "should be a string" unless timezone.is_a?(String)
+        error "unknown time zone #{timezone}, use an IANA name such as Europe/Stockholm" if ActiveSupport::TimeZone[timezone].nil?
+      end
+    end
+
+    def validate_source!(source, block)
+      with_context(source) do
+        validate_type! block, Hash
+        check_unknown_keys! block, (source == "prometheus" ? %w[ url via token ] : %w[ url via ]).index_with(nil)
+
+        with_context("url") do
+          validate_secret_or_string! block["url"]
+          url = block["url"]
+          if url.is_a?(String)
+            error "should start with #{SOURCES[source].map { |scheme| "#{scheme}://" }.join(" or ")}" if SOURCES[source].none? { |scheme| url.start_with?("#{scheme}://") }
+            valid_uri!(url)
+            # Postgres may name a socket instead (postgres:///app?host=/run/postgresql).
+            error "should include a host" if source == "prometheus" && URI(url).host.blank?
+          end
+        end
+
+        with_context("via") do
+          via = block["via"]
+          error "should be a host name" unless via.nil? || (via.is_a?(String) && via.present? && via == via.strip)
+        end
+
+        with_context("token") { validate_secret_or_string! block["token"] } if block.key?("token")
+
+        # Both checked again for a secret URL: rediss when it is resolved
+        # (Dash::Configuration::Autoscale#source_url), the host when the tunnel opens.
+        if block["via"] && Dash::Configuration::Autoscale.unverifiable_through_tunnel?(source, block["url"])
+          error Dash::Configuration::Autoscale::REDISS_VIA_MESSAGE
+        end
+        error "via needs a URL that names its host" if block["via"] && block["url"].is_a?(String) && URI(block["url"]).host.blank?
+      end
+    end
+
+    def valid_uri!(url)
+      URI(url)
+    rescue URI::InvalidURIError
+      error "is not a valid URL"
+    end
+
+    def validate_secret_or_string!(value)
+      error "is required" if value.blank? || (value.is_a?(Array) && value.first.blank?)
+
+      unless value.is_a?(String) || (value.is_a?(Array) && value.size == 1 && value.first.is_a?(String))
+        error "should be a string or an array with one string (for secret lookup)"
+      end
+    end
+
+    def validate_at_least!(value, minimum, unit = nil)
+      return if value.nil?
+
+      error "should be an integer" unless value.is_a?(Integer)
+      error "must be at least #{minimum}#{unit}, not #{value}" if value < minimum
+    end
+
+    def validate_upcloud!(upcloud)
+      with_context("upcloud") do
+        validate_type! upcloud, Hash
+        check_unknown_keys! upcloud, CREDENTIALS.index_with(nil)
+
+        CREDENTIALS.each do |key|
+          with_context(key) { validate_secret_or_string! upcloud[key] }
+        end
+      end
+    end
+
+    def validate_exec!(exec)
+      with_context("exec") do
+        validate_type! exec, Hash
+        check_unknown_keys! exec, SCRIPTS.index_with(nil)
+
+        exec.each { |key, value| with_context(key) { validate_type! value, String } }
+        REQUIRED_SCRIPTS.each { |key| with_context(key) { error "is required" if exec[key].blank? } }
+      end
+    end
+end

data/lib/dash/configuration/validator/proxy.rb

@@ -645,6 +645,7 @@ class Dash::Configuration::Validator::Proxy < Dash::Configuration::Validator
         end
 
         validate_min_tls! run_config["min_tls"]
+        validate_run_version! run_config["version"]
 
         RUN_TIMEOUTS.each do |key|
           error "#{key} cannot be negative" if run_config[key].to_f.negative?
@@ -673,6 +674,18 @@ class Dash::Configuration::Validator::Proxy < Dash::Configuration::Validator
       end
     end
 
+    # `dash proxy boot` refuses a running proxy older than MINIMUM_VERSION, and a
+    # reboot would start the pinned tag again. A tag that is not a version number
+    # (a branch build) cannot be compared and is left to the operator.
+    def validate_run_version!(value)
+      minimum = Dash::Configuration::Proxy::Run::MINIMUM_VERSION
+      return unless value.present? && Gem::Version.correct?(value.to_s.delete_prefix("v"))
+
+      if Dash::Utils.older_version?(value.to_s, minimum)
+        error "version #{value} is older than #{minimum}, the oldest dash-proxy this dash runs - remove the pin or raise it"
+      end
+    end
+
     # dash-proxy reads the cache policy only when --cache is set and ignores it
     # otherwise, so a tuned block with no `enabled` is a cache that silently
     # never caches - this issue's own predicted first support question.

data/lib/dash/configuration/validator/role.rb

@@ -1,4 +1,9 @@
 class Dash::Configuration::Validator::Role < Dash::Configuration::Validator
+  # The reactive autoscaling keys whose values the example cannot type: a threshold may be
+  # an integer or a float, a source decides which keys apply. Dash::Configuration::Role::Scale
+  # validates them in full.
+  SCALE_RULES = %w[ signal up down ]
+
   def validate!
     validate_type! config, Array, Hash
 
@@ -16,12 +21,28 @@ class Dash::Configuration::Validator::Role < Dash::Configuration::Validator
     # `healthcheck: false` is the explicit opt-out from the readiness gate, so the
     # example's hash shape is not the only legal one.
     def validate_key_override!(key, value)
-      return false unless key.to_s == "healthcheck"
+      case key.to_s
+      when "healthcheck"
+        case value
+        when false then true
+        when Hash then false
+        else error "should be a hash, or false to accept no readiness gate for this role"
+        end
+      when "replicas"
+        # `replicas: 2` is shorthand for min and max, so the example's hash shape is not
+        # the only legal one either.
+        case value
+        when Integer then true
+        when Hash then false
+        else error "should be an integer, or a hash with min and max"
+        end
+      when *SCALE_RULES
+        return false unless context.end_with?("scale/#{key}")
 
-      case value
-      when false then true
-      when Hash then false
-      else error "should be a hash, or false to accept no readiness gate for this role"
+        validate_type! value, Hash
+        true
+      else
+        false
       end
     end