dash 4.1.3 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17c99f41669956c79ffd4272e4fd524171c4ff9a37f26d885bd4b4cdc084be85
-  data.tar.gz: 8fa42e665988fd8d343b0d0141438096635e6059566767bb3d63375fe975e324
+  metadata.gz: e8fe407ebe24b10effd6d4236b4e9035f6a01362098e6b23a97d5fc120c6e9b5
+  data.tar.gz: f10f2b259bd849dde14c9bdf2982571f20cbafe201141a4a4a797eac14ce25ee
 SHA512:
-  metadata.gz: c12f97e422182fba7128f2c27094dc824b0c76825273b89f70d10563c4acc6a59a1d22fe10c22c47b7e0456a6a50c30de0343a6c42de1e0ebd677e5a602c19f7
-  data.tar.gz: 897652d67bb727025f490d7ee8ca1b719c77c13a2fb896826742c6fc34f5ff3968384925dccd01e00d6493a07c98e75212e76933c97d36323e03c20fb3436d46
+  metadata.gz: dcf7d8731f2526798b9e601a9c75c81c728fd7a203a5191fad0eb61473ac2cf474c53de4c0efb83e640610ba3674969d58bc8f8033f7e46f5595bfdc43a83373
+  data.tar.gz: 00a5d2988465fdaf6b5df1e0beca380b92ab20ccc61a00cda1fdab9030c44b1c50e9e763a4736d1e649dfc99d3c2020a173ceff58df25118a0522f03b1778771

data/lib/dash/autoscale/connections/builder.rb

@@ -0,0 +1,51 @@
+# How each source's client is made. redis-client and pg are not dash dependencies: they are
+# required here, on first use, and a missing one names the gem to add to the Gemfile the
+# controller runs with.
+module Dash::Autoscale::Connections::Builder
+  GEMS = { redis: "redis-client", postgres: "pg" }.freeze
+  CONNECT_TIMEOUT = 2
+  READ_TIMEOUT = 5
+  POSTGRES_CONNECT_TIMEOUT = 5
+
+  def self.build(source, url, tunnel, config:)
+    case source
+    when :redis
+      load_gem(source)
+      RedisClient.new(**redis_options(url, tunnel))
+    when :postgres
+      load_gem(source)
+      PG.connect(postgres_options(conninfo(url), tunnel))
+    when :prometheus
+      Dash::Autoscale::PrometheusClient.new(url: url, token: config.autoscale.prometheus_token, tunnel: tunnel)
+    end
+  end
+
+  # redis-client takes host and port over the URL's, and keeps the rest (password, db).
+  def self.redis_options(url, tunnel)
+    { url: url, **({ host: tunnel.address, port: tunnel.port } if tunnel).to_h,
+      connect_timeout: CONNECT_TIMEOUT, read_timeout: READ_TIMEOUT, write_timeout: READ_TIMEOUT }
+  end
+
+  # libpq connects to `hostaddr` and verifies TLS against `host`, so a tunnel only sets
+  # hostaddr and port.
+  def self.postgres_options(conninfo, tunnel)
+    conninfo.merge(**({ hostaddr: tunnel.address, port: tunnel.port } if tunnel).to_h, connect_timeout: POSTGRES_CONNECT_TIMEOUT)
+  end
+
+  # The URL as libpq reads it, so nothing in it is lost when a tunnel changes where it goes.
+  def self.conninfo(url)
+    PG::Connection.conninfo_parse(url).each_with_object({}) { |option, hash| hash[option[:keyword].to_sym] = option[:val] if option[:val] }
+  end
+
+  # Only the gem itself missing means "add it"; an installed one that fails to load (a
+  # native library, a dependency) says why. Both are configuration errors: a LoadError is
+  # not a StandardError, and would end the controller.
+  def self.load_gem(source)
+    require GEMS.fetch(source)
+  rescue LoadError => e
+    gem = GEMS.fetch(source)
+    raise Dash::ConfigurationError, "autoscale/#{source}: the #{gem} gem failed to load: #{e.message}" unless e.path == gem
+
+    raise Dash::ConfigurationError, %(autoscale/#{source}: needs the #{gem} gem - add gem "#{gem}" to the Gemfile dash autoscale runs with)
+  end
+end

data/lib/dash/autoscale/connections.rb

@@ -0,0 +1,102 @@
+require "net/ssh"
+require "timeout"
+
+# The controller's clients for its signals and guards - Redis, Postgres, Prometheus - one per
+# source for the whole process, so they outlive `reconfigure!` and every tick reuses them.
+# Each is built on first use, through a Dash::Autoscale::Tunnel when the source has `via`,
+# and rebuilt after an error, when its URL, via or Prometheus token changes, or when its tunnel died.
+#
+# One read at a time per source: neither redis-client nor pg connections are thread-safe,
+# and the drain's busy polls (inside SSHKit's `on` threads) share them with the tick.
+class Dash::Autoscale::Connections
+  DEFAULT_PORTS = { redis: 6379, postgres: 5432 }.freeze
+
+  # Errors that leave a client (or the tunnel under it) unusable. Anything else - a query
+  # that failed, a malformed job, an HTTP error status - keeps it: the next tick reuses it.
+  # The drivers' classes are named, since their gems may not be loaded.
+  CONNECTION_ERRORS = [ IOError, SystemCallError, SocketError, Timeout::Error, Net::SSH::Exception ].freeze
+  DRIVER_CONNECTION_ERRORS = %w[ RedisClient::ConnectionError RedisClient::ProtocolError PG::ConnectionBad PG::UnableToSend ].freeze
+
+  Entry = Struct.new(:key, :client, :tunnel)
+
+  # The process's connections: the controller's, and the drain's when `dash scale set` runs
+  # inside it. Closed by the controller when it stops; a later read opens them again.
+  def self.shared
+    @shared ||= new
+  end
+
+  def initialize(config: -> { DASH.config }, builder: Builder.method(:build), tunnel: Dash::Autoscale::Tunnel.method(:open))
+    @config, @builder, @tunnel = config, builder, tunnel
+    @entries = {}
+    @locks = Hash.new { |locks, source| locks[source] = Mutex.new }
+    @locks_lock = Mutex.new
+  end
+
+  def with(source)
+    source = source.to_sym
+
+    lock(source).synchronize do
+      entry = entry_for(source)
+
+      begin
+        yield entry.client
+      rescue StandardError => e
+        discard(source) if connection_error?(e)
+        raise
+      end
+    end
+  end
+
+  def close
+    @entries.keys.each { |source| lock(source).synchronize { discard(source) } }
+  end
+
+  private
+    def lock(source)
+      @locks_lock.synchronize { @locks[source] }
+    end
+
+    # A client is kept while what it was built from - the URL, via and, for Prometheus, the
+    # token it sends - stays the same.
+    def entry_for(source)
+      autoscale = @config.call.autoscale
+      url, via = autoscale.source_url(source), autoscale.source_via(source)
+      key = [ url, via, (autoscale.prometheus_token if source == :prometheus) ]
+      entry = @entries[source]
+      return entry if entry && entry.key == key && (entry.tunnel.nil? || entry.tunnel.alive?)
+
+      discard(source)
+      @entries[source] = build(source, url, via, key)
+    end
+
+    def build(source, url, via, key)
+      uri = URI(url)
+      raise Dash::ConfigurationError, "autoscale/#{source}: via needs a URL that names its host" if via && uri.host.blank?
+
+      tunnel = @tunnel.call(via, uri.host, uri.port || DEFAULT_PORTS.fetch(source)) if via
+      Entry.new(key, @builder.call(source, url, tunnel, config: @config.call), tunnel)
+    rescue StandardError
+      tunnel&.close
+      raise
+    end
+
+    def connection_error?(error)
+      CONNECTION_ERRORS.any? { |type| error.is_a?(type) } || error.class.ancestors.any? { |type| DRIVER_CONNECTION_ERRORS.include?(type.name) }
+    end
+
+    def discard(source)
+      entry = @entries.delete(source)
+      return unless entry
+
+      close_quietly(entry.client)
+      close_quietly(entry.tunnel)
+    end
+
+    # A client or tunnel being thrown away after an error may fail to close; the error
+    # that matters has been raised already.
+    def close_quietly(closable)
+      closable&.close
+    rescue StandardError
+      nil
+    end
+end

data/lib/dash/autoscale/controller.rb

@@ -0,0 +1,296 @@
+require "securerandom"
+require "socket"
+
+# `dash autoscale run`: every `autoscale.interval` seconds, for each scaled role with a
+# `scale.schedule` or `scale.signal`, observe it, ask Dash::Autoscale::Policy, and carry the decision out
+# through `dash scale set` (or replace an unreachable member) under the deploy lock -
+# never by building a join itself.
+#
+# Its state lives on the state host (Dash::Autoscale::StateStore) and is read at the
+# start of every tick, so a restart or a `--once` run from cron behaves like the long-lived
+# loop. The heartbeat is the lease: a second controller refuses to start while it is
+# alive, and this one stops when it names another controller.
+#
+# A failure in one role becomes that role's `action_failed` (or `lock_busy`) hold and the
+# tick moves on. A tick that fails as a whole (the state host down) is reported and the
+# loop goes on; only a lost lease or a deploy.yml that no longer loads stops it.
+class Dash::Autoscale::Controller
+  include SSHKit::DSL
+
+  DECISION_LOG_LIMIT = 10_000
+  TRIM_CHECK_INTERVAL = 3600
+
+  # One scaled role on one tick: the state it is carried in, how it was evaluated, and
+  # what was decided (or what became of the decision once carried out).
+  Entry = Struct.new(:role, :state, :evaluation, :decision, keyword_init: true)
+
+  attr_reader :id, :metrics, :cli
+
+  # `cli` reconfigures DASH, runs `dash scale set`, replaces members and prints - see
+  # Dash::Cli::Autoscale.
+  # `reader` reads the signals and guards, over connections the controller closes when it
+  # stops - see Dash::Autoscale::Reader.
+  def initialize(cli:, clock: -> { Time.now.utc }, sleeper: ->(seconds) { sleep seconds }, dry_run: false, takeover: false,
+    interval: nil, mode: "loop", recorder: nil, metrics: Dash::Autoscale::Metrics.new, reader: Dash::Autoscale::Reader.new)
+    @cli, @clock, @sleeper = cli, clock, sleeper
+    @dry_run, @takeover, @interval, @mode = dry_run, takeover, interval, mode
+    @recorder, @metrics, @reader = recorder, metrics, reader
+    @id = SecureRandom.hex(8)
+    @stopping = false
+  end
+
+  def dry_run?
+    @dry_run
+  end
+
+  def interval
+    @interval || DASH.config.autoscale.interval
+  end
+
+  # The roles the controller owns the count of: scaled, with a schedule or a signal, within --roles.
+  def roles
+    (DASH.specific_roles || DASH.config.roles).select(&:controlled?)
+  end
+
+  # Claims the lease, unless another controller's is alive.
+  def start
+    cli.reconfigure!
+    raise ArgumentError, "No scaled role in scope has a scale/schedule or a scale/signal, so there is nothing for the controller to do" if roles.empty?
+
+    @started_at = @clock.call
+    return if dry_run?
+
+    # Raised out here: SSHKit would wrap anything raised inside `on`.
+    lease = with_store { |store| store.ensure_directory; Dash::Autoscale::Lease.new(store.heartbeat, now: @started_at) }
+    if lease.held_by_another?(id) && !@takeover
+      raise Dash::Autoscale::LeaseHeld, "Another autoscale controller is running: #{lease.describe}. Stop it, or start this one with --takeover"
+    end
+
+    @role_names = roles.map(&:name)
+    with_store { |store| store.write_heartbeat(heartbeat(@started_at)) }
+  end
+
+  def run
+    start
+
+    until @stopping
+      tick_started = @clock.call
+      tick_reporting_failure
+      wait_until(tick_started + interval)
+    end
+  ensure
+    release
+  end
+
+  # From a signal handler: only a flag, the current tick (and a join in flight) finishes.
+  def stop!
+    @stopping = true
+  end
+
+  def stopping?
+    @stopping
+  end
+
+  def tick
+    started = monotonic
+    now = @clock.call
+    cli.reconfigure!
+
+    heartbeat, state, pauses = read_state
+    ensure_lease(heartbeat)
+
+    entries = roles.map { |role| evaluate(role, now, state, pauses) }
+    unless dry_run?
+      acting = entries.any? { |entry| !entry.decision.hold? }
+      keeping_heartbeat { execute(entries) } if acting
+      persist(entries, state, now)
+    end
+
+    decisions = entries.map(&:decision)
+    @metrics.record_tick(decisions: decisions, at: @clock.call, duration: monotonic - started,
+      members: entries.to_h { |entry| [ entry.role.name, entry.evaluation&.observation&.pool.to_a.map(&:state) ] })
+    @recorder&.record(now, decisions)
+    decisions.each { |decision| cli.report decision.summary, (:red if decision.error) }
+    decisions
+  end
+
+  # Closes the signal connections and marks the heartbeat stopped, so the next controller
+  # does not wait out a stale lease - unless another controller holds it by now.
+  def release
+    @reader.close
+    return if dry_run? || @started_at.nil?
+
+    with_store { |store| store.write_heartbeat_if_held(id, heartbeat(@last_tick_at || @started_at).merge(stopped_at: @clock.call)) }
+  rescue StandardError => e
+    cli.report "Could not mark the autoscale controller stopped: #{e.class}: #{e.message}", :yellow
+  end
+
+  private
+    def tick_reporting_failure
+      tick
+    rescue Dash::Autoscale::LeaseLost, Dash::ConfigurationError
+      raise
+    rescue StandardError => e
+      cli.report "Autoscale tick failed: #{e.class}: #{e.message}", :red
+    end
+
+    def wait_until(deadline)
+      while !@stopping && (remaining = deadline - @clock.call) > 0
+        @sleeper.call([ remaining, 1 ].min)
+      end
+    end
+
+    def read_state
+      quietly do
+        with_store { |store| [ store.heartbeat, store.state, store.pauses ] }
+      end
+    end
+
+    def ensure_lease(heartbeat)
+      return if dry_run? || heartbeat["controller_id"].blank? || heartbeat["controller_id"] == id
+
+      raise Dash::Autoscale::LeaseLost, "Another autoscale controller took over: #{Dash::Autoscale::Lease.new(heartbeat, now: @clock.call).describe}"
+    end
+
+    def evaluate(role, now, state, pauses)
+      entry = Entry.new(role: role, state: Dash::Autoscale::RoleState.from(state.dig("roles", role.name)))
+      entry.evaluation = Dash::Autoscale::Evaluation.new(role: role, now: now, state: entry.state, pause: Dash::Autoscale::Pause.from(pauses[role.name]),
+        observation: quietly { Dash::Autoscale::Observation.take(role) }, reader: @reader)
+      entry.decision = entry.evaluation.decide
+      entry
+    rescue StandardError => e
+      entry.evaluation = nil
+      entry.decision = Dash::Autoscale::Decision.new(role: role.name, action: "hold", reasons: [ "action_failed" ], at: now, error: "#{e.class}: #{e.message}")
+      entry
+    end
+
+    # Taking the deploy lock first ensures the run directory on every host of the app, and
+    # a host that did not answer this tick would fail that for every role. The hosts this
+    # tick saw down are skipped for it (the trick Dash::Cli::Scale#replace_unreachable uses
+    # for its own member).
+    def execute(entries)
+      skipped = entries.flat_map { |entry| entry.evaluation&.observation&.unread_hosts.to_a }.uniq - DASH.run_directory_ensured_on
+      DASH.run_directory_ensured_on.concat(skipped)
+
+      entries.each { |entry| entry.decision = carry_out(entry) unless entry.decision.hold? }
+    ensure
+      skipped&.each { |host| DASH.run_directory_ensured_on.delete(host) }
+    end
+
+    # A join can take minutes (lock wait, boot_timeout, the app's boot). The heartbeat keeps
+    # beating meanwhile, so the lease stays alive and status does not read the controller as
+    # stuck exactly while it scales. A beat that finds another controller's id stops beating.
+    def keeping_heartbeat
+      done = false
+      beat = Thread.new do
+        until done
+          Kernel.sleep(interval)
+          break if done || !beat_heartbeat
+        end
+      end
+
+      yield
+    ensure
+      done = true
+      beat&.wakeup if beat&.alive?
+      # A beat still writing should land before anything after it (the tick's own heartbeat,
+      # or release's stopped_at), but a write that hangs must not hold the tick: past two
+      # intervals the beat is killed.
+      beat.kill unless beat.nil? || beat.join(2 * interval)
+    end
+
+    def beat_heartbeat
+      with_store { |store| store.write_heartbeat_if_held(id, heartbeat(@clock.call)) }
+    rescue StandardError
+      true # a missed beat is retried on the next one
+    end
+
+    def carry_out(entry)
+      role, decision, observation = entry.role, entry.decision, entry.evaluation.observation
+
+      case decision.action
+      when "scale_out", "scale_in"
+        cli.invoke_scale_set(role, decision.to)
+        entry.state.scaled(decision.action, now: @clock.call)
+        record_joins(entry, observation.member_hosts) if decision.action == "scale_out"
+      when "replace_member"
+        member = observation.members.find { |candidate| candidate.host == decision.member_host }
+        cli.replace_member(role, member, count: decision.from)
+        entry.state.forget(member.host)
+      end
+
+      decision
+    rescue Dash::Cli::LockError => e
+      failed(decision, "lock_busy", e.message)
+    rescue StandardError => e
+      failed(decision, "action_failed", "#{e.class}: #{e.message}")
+    end
+
+    # The members a scale-out powered on warm up from now. Members someone else joined are
+    # not known to be warming, so they never hold a scale-in.
+    #
+    # The scale-out already happened, so a pool that cannot be read now is only reported:
+    # the members it joined go without a warmup rather than the action being logged failed.
+    def record_joins(entry, before)
+      DASH.config.pool.refresh!
+      entry.state.joined_at(entry.role.active_members.map(&:host) - before, now: @clock.call)
+    rescue StandardError => e
+      cli.report "#{entry.role}: scaled out, but the members it joined could not be read for their warmup (#{e.class}: #{e.message})", :yellow
+    end
+
+    def failed(decision, reason, message)
+      Dash::Autoscale::Decision.new(role: decision.role, action: "hold", from: decision.from, to: decision.from, reasons: [ reason ],
+        inputs: decision.inputs.merge(attempted: { action: decision.action, to: decision.to, replace: decision.member_host }.compact),
+        eligible_at: decision.eligible_at, at: decision.at, error: message)
+    end
+
+    def persist(entries, state, now)
+      logged = entries.select { |entry| entry.state.log?(entry.decision) }.each { |entry| entry.state.logged(entry.decision) }.map(&:decision)
+      roles_state = state.fetch("roles", {}).merge(entries.to_h { |entry| [ entry.role.name, entry.state.to_h ] })
+      checked_at = Dash::Autoscale::Timestamp.parse(state["decisions_checked_at"])
+      check = checked_at.nil? || now - checked_at >= TRIM_CHECK_INTERVAL
+
+      # The heartbeat first, and only while it still names this controller: one command on
+      # the host checks and writes, so a takeover since the tick started (or while it acted)
+      # is never overwritten, and its state is left to the controller that took over.
+      beat_at = @clock.call
+      held = quietly { with_store { |store| store.write_heartbeat_if_held(id, heartbeat(beat_at)) } }
+      # Only a heartbeat naming another controller is a lost lease; a write that failed
+      # raised above and is this tick's failure, which the loop survives.
+      raise Dash::Autoscale::LeaseLost, "Another autoscale controller took over while this one ticked" unless held
+
+      @last_tick_at = beat_at
+      quietly do
+        with_store do |store|
+          store.append_decisions(logged)
+          trim(store) if check
+          updated = state.merge("roles" => roles_state, "decisions_checked_at" => Dash::Autoscale::Timestamp.dump(check ? now : checked_at))
+          store.write_state(updated) unless updated == state
+        end
+      end
+    end
+
+    def trim(store)
+      store.trim_decisions(keep: DECISION_LOG_LIMIT) if store.decision_count > DECISION_LOG_LIMIT
+    end
+
+    def heartbeat(last_tick_at)
+      { controller_id: id, hostname: Socket.gethostname, pid: Process.pid, version: Dash::VERSION, mode: @mode,
+        started_at: @started_at, last_tick_at: last_tick_at, interval: interval, roles: @role_names }
+    end
+
+    def with_store(&block)
+      result = nil
+      on(Dash::Autoscale::StateStore.host) { result = block.call(Dash::Autoscale::StateStore.new(self)) }
+      result
+    end
+
+    # The controller reads every host every tick; those reads print only with --verbose.
+    def quietly(&block)
+      DASH.verbosity == :debug ? block.call : DASH.with_verbosity(:error, &block)
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+end

data/lib/dash/autoscale/cron.rb

@@ -0,0 +1,75 @@
+# A five-field cron expression - `minute hour day-of-month month day-of-week` - for the
+# schedule windows of a scaled role. Each field is `*`, a number, a range `a-b`, a list
+# `a,b` or a step `*/n` / `a-b/n`; day-of-week 0 and 7 are both Sunday. Vixie semantics:
+# when both day fields are restricted (neither starts with `*`), a day matches if either
+# does. No names, `L`, `W` or `#`.
+#
+# `match?` takes a time already in the zone the schedule is evaluated in.
+class Dash::Autoscale::Cron
+  class Error < ArgumentError; end
+
+  Field = Struct.new(:name, :min, :max)
+
+  FIELDS = [
+    Field.new("minute", 0, 59),
+    Field.new("hour", 0, 23),
+    Field.new("day-of-month", 1, 31),
+    Field.new("month", 1, 12),
+    Field.new("day-of-week", 0, 7)
+  ].freeze
+
+  ITEM = /\A(?:(\*)|(\d+)(?:-(\d+))?)(?:\/(\d+))?\z/
+
+  def self.parse(expression)
+    new(expression)
+  end
+
+  def initialize(expression)
+    @expression = expression.to_s.strip
+    parts = @expression.split
+
+    unless parts.size == FIELDS.size
+      raise Error, "#{@expression.inspect} has #{parts.size} fields, a cron has 5 (minute hour day-of-month month day-of-week)"
+    end
+
+    @minutes, @hours, @days, @months, weekdays = FIELDS.zip(parts).map { |field, part| parse_field(field, part) }
+    @weekdays = weekdays.map { |day| day % 7 }.to_set
+    @days_restricted, @weekdays_restricted = !parts[2].start_with?("*"), !parts[4].start_with?("*")
+  end
+
+  def match?(time)
+    @minutes.include?(time.min) && @hours.include?(time.hour) && @months.include?(time.month) && day_match?(time)
+  end
+
+  def to_s
+    @expression
+  end
+
+  private
+    def day_match?(time)
+      day, weekday = @days.include?(time.day), @weekdays.include?(time.wday)
+
+      @days_restricted && @weekdays_restricted ? day || weekday : day && weekday
+    end
+
+    def parse_field(field, part)
+      part.split(",", -1).flat_map { |item| parse_item(field, item) }.to_set
+    end
+
+    def parse_item(field, item)
+      match = ITEM.match(item)
+      unless match && (match[1] || match[3] || !match[4])
+        raise Error, "#{field.name}: #{item} is not supported, use numbers, *, ranges (a-b), lists (a,b) and steps (*/n, a-b/n)"
+      end
+
+      star, first, last, step = match.captures
+      from, to = star ? [ field.min, field.max ] : [ Integer(first, 10), Integer(last || first, 10) ]
+      step = step ? Integer(step, 10) : 1
+
+      [ from, to ].each { |value| raise Error, "#{field.name}: #{value} is outside #{field.min}-#{field.max}" unless value.between?(field.min, field.max) }
+      raise Error, "#{field.name}: #{item} is a reversed range" if from > to
+      raise Error, "#{field.name}: #{item} needs a step of at least 1" if step < 1
+
+      (from..to).step(step).to_a
+    end
+end

data/lib/dash/autoscale/decision.rb

@@ -0,0 +1,89 @@
+# What the controller decided for one scaled role on one tick: the action, the container
+# count it goes from and to, the reason codes, and every input the policy used. The reason
+# and action codes are stable - the decision log, the metrics and the MCP tools all use them.
+#
+# On a scale_out or scale_in the reasons name what set the target the role moves toward:
+# `schedule_floor` (a window), `at_min` (scale min × replicas min), plus `at_max` when the
+# target was capped. A scale_in step that stops short of the target still names the target.
+# A signal-driven step names its signal instead: `above_threshold` or `below_threshold`.
+Dash::Autoscale::Decision = Struct.new(:role, :action, :from, :to, :reasons, :inputs, :eligible_at, :at, :error, keyword_init: true) do
+  def initialize(reasons: [], inputs: {}, **attributes)
+    super
+  end
+
+  # Times as ISO 8601 in UTC, symbols as strings, all the way down - what JSON, the MCP
+  # tools and the decision log carry.
+  def self.json_safe(value)
+    case value
+    when Hash then value.to_h { |key, item| [ key, json_safe(item) ] }
+    when Array then value.map { |item| json_safe(item) }
+    when Time, ActiveSupport::TimeWithZone then value.utc.iso8601
+    when Symbol then value.to_s
+    when Exception then value.message
+    else value
+    end
+  end
+
+  def hold?
+    action == "hold"
+  end
+
+  # The member a `replace_member` decision takes out.
+  def member_host
+    inputs[:replace]
+  end
+
+  def to_h
+    Dash::Autoscale::Decision.json_safe(super)
+  end
+
+  # One log line: `payments: scale_out 4 -> 10 (schedule_floor: window "0 22 1,15 * *" min 10)`.
+  def summary
+    counts = case action
+    when "hold" then "at #{from.nil? ? "?" : from}"
+    when "replace_member" then member_host
+    else "#{from} -> #{to}"
+    end
+
+    "#{role}: #{action} #{counts} (#{reasons.map { |reason| note(reason) }.join(", ")})"
+  end
+
+  private
+    # `above_threshold: 12.4 > 10 for 45s`, or `... until <time>` while it has not been long enough.
+    def threshold(reason)
+      above = reason == "above_threshold"
+      since = inputs[above ? :above_since : :below_since]
+      text = "#{reason}: #{inputs.dig(:signal, :value)} #{above ? ">" : "<"} #{inputs[above ? :up_above : :down_below]}"
+
+      if hold? && eligible_at
+        "#{text} until #{eligible_at.utc.iso8601}"
+      elsif since
+        "#{text} for #{(at - since).round}s"
+      else
+        text
+      end
+    end
+
+    def note(reason)
+      case reason
+      when "schedule_floor"
+        window = inputs[:windows].to_a.max_by { |candidate| candidate[:min] }
+        window ? %(#{reason}: window "#{window[:cron]}" min #{window[:min]}) : reason
+      when "cooldown", "warming_up", "member_unreachable", "paused"
+        eligible_at ? "#{reason} until #{eligible_at.utc.iso8601}" : reason
+      when "above_threshold", "below_threshold"
+        threshold(reason)
+      when "held_by_guard"
+        "#{reason}: #{inputs[:held_by]}"
+      else
+        error ? "#{reason}: #{error}" : reason
+      end
+    end
+end
+
+Dash::Autoscale::Decision::ACTIONS = %w[ scale_out scale_in replace_member hold ].freeze
+Dash::Autoscale::Decision::REASONS = %w[
+  schedule_floor at_min at_max at_target cooldown warming_up paused lock_busy
+  member_unreachable host_unreachable pool_unreadable action_failed
+  above_threshold below_threshold held_by_guard missing_data
+].freeze

data/lib/dash/autoscale/duration.rb

@@ -0,0 +1,19 @@
+# A length of time as deploy.yml and the CLI write it: integer seconds, or a whole number
+# with a unit - `90s`, `15m`, `30h`, `7d`. Returns seconds.
+module Dash::Autoscale::Duration
+  UNITS = { "s" => 1, "m" => 60, "h" => 3600, "d" => 86_400 }.freeze
+  FORMAT = /\A(\d+)([smhd]?)\z/
+
+  def self.parse(value)
+    case value
+    when Integer
+      return value unless value.negative?
+    when String
+      if (match = FORMAT.match(value))
+        return Integer(match[1], 10) * UNITS.fetch(match[2].presence || "s")
+      end
+    end
+
+    raise ArgumentError, "#{value.inspect} should be seconds, or a number followed by s, m, h or d (30m, 2h, 1d)"
+  end
+end