dash 4.1.2 → 4.2.0

data/lib/dash/autoscale/evaluation.rb

@@ -0,0 +1,52 @@
+# One policy evaluation for one scaled role: observes it (unless handed an observation),
+# reads its signal and guards when it has a signal, brings the role's remembered state up to
+# date with what it saw - unreachable clocks started or cleared, warmups of members that left
+# dropped, breach windows moved - and asks Dash::Autoscale::Policy.
+# Takes no action and writes nothing: the controller's tick and `dash autoscale explain`
+# both call this, so explain answers exactly what the next tick would decide.
+class Dash::Autoscale::Evaluation
+  attr_reader :role, :now, :state, :observation
+
+  def initialize(role:, now:, state:, pause:, observation: nil, reader: nil)
+    @role, @now, @state, @pause = role, now, state, pause
+    @observation = observation || Dash::Autoscale::Observation.take(role)
+    @reader = reader
+  end
+
+  def decide
+    # With the pool unreadable nothing is known about the members, so nothing is forgotten.
+    unless observation.pool_error
+      state.observe_unreachable(observation.unreachable_members.keys, now: now)
+      state.keep_members(observation.member_hosts)
+    end
+
+    signal, guards, configuration_error = read_signal
+
+    decision = Dash::Autoscale::Policy.new(role: role, now: now, time_zone: role.config.autoscale.time_zone, current: observation.current,
+      history: state.history, paused_until: @pause&.ends_at, warming: state.joined, unreachable: state.unreachable,
+      unreadable_baseline: observation.unreadable_baseline, pool_error: observation.pool_error,
+      signal: signal, guards: guards, breach: state.breach).decide
+
+    # A signal that cannot be configured (a missing gem, an empty secret) fails the role's
+    # evaluation loudly - but only where the signal decides: a floor, a pause or an
+    # unreachable member goes on without it.
+    raise configuration_error if configuration_error && decision.reasons == [ "missing_data" ]
+
+    decision
+  end
+
+  private
+    def read_signal
+      return [ nil, [] ] unless role.scale.signal?
+
+      signal, guards = begin
+        (@reader || Dash::Autoscale::Reader.new).read(role, at: now)
+      rescue Dash::ConfigurationError => e
+        error = e
+        [ Dash::Autoscale::Sample.new(value: nil, at: now, error: "#{e.class}: #{e.message}"), [] ]
+      end
+
+      state.observe_signal(signal, up_above: role.scale.up_above, down_below: role.scale.down_below)
+      [ signal, guards, error ]
+    end
+end

data/lib/dash/autoscale/guard.rb

@@ -0,0 +1,48 @@
+# A `scale.hold_when` guard as the controller reads it. A Reading pairs the guard (its index,
+# source and bound) with its Sample: it holds a signal-driven scale-out while its value is
+# above the bound, or while it cannot be read.
+class Dash::Autoscale::Guard
+  # One Reading per guard, in order. A guard that cannot be read is a Reading with an error,
+  # which holds; only a configuration error goes through.
+  def self.read(guards, connections, at: Time.now.utc)
+    guards.each_with_index.map do |guard, index|
+      Reading.new(index: index, source: guard.source, above: guard.above, sample: sample(guard, connections, at))
+    end
+  end
+
+  def self.sample(guard, connections, at)
+    value = connections.with(guard.connection) do |client|
+      if guard.source == "postgres"
+        Dash::Autoscale::ReadOnlyQuery.within(client) { Dash::Autoscale::ReadOnlyQuery.number(client, guard.query) }
+      else
+        client.query(guard.query)
+      end
+    end
+
+    Dash::Autoscale::Sample.new(value: value, at: at)
+  rescue Dash::ConfigurationError
+    raise
+  rescue StandardError => e
+    Dash::Autoscale::Sample.new(value: nil, at: at, error: "#{e.class}: #{e.message}")
+  end
+  private_class_method :sample
+
+  Reading = Struct.new(:index, :source, :above, :sample, keyword_init: true) do
+    def holding?
+      sample.missing? || sample.value > above
+    end
+
+    def name
+      "hold_when/#{index} #{source}"
+    end
+
+    # `hold_when/0 postgres 195 > 180`, or `hold_when/0 postgres: <error>`.
+    def describe
+      sample.missing? ? "#{name}: #{sample.error || "no value"}" : "#{name} #{sample.value} > #{above}"
+    end
+
+    def to_h
+      { index: index, source: source, above: above }.merge(sample.to_h)
+    end
+  end
+end

data/lib/dash/autoscale/lease.rb

@@ -0,0 +1,55 @@
+# The controller's heartbeat read as a lease: alive while its last tick is within three of
+# its own intervals and it has not stopped. A second controller refuses to start while
+# another one's lease is alive, and a running one stops when the heartbeat names another
+# controller. Convergent, not a hard mutex - the deploy lock still serializes every scale
+# action.
+class Dash::Autoscale::Lease
+  STALE_AFTER_INTERVALS = 3
+
+  attr_reader :heartbeat
+
+  def initialize(heartbeat, now:)
+    @heartbeat = heartbeat.is_a?(Hash) ? heartbeat : {}
+    @now = now
+  end
+
+  def controller_id
+    heartbeat["controller_id"]
+  end
+
+  def last_tick_at
+    Dash::Autoscale::Timestamp.parse(heartbeat["last_tick_at"])
+  end
+
+  def interval
+    Integer(heartbeat["interval"].to_s, 10)
+  rescue ArgumentError
+    Dash::Configuration::Autoscale::DEFAULT_INTERVAL
+  end
+
+  # Whole seconds since the last tick, nil without one.
+  def age
+    (@now - last_tick_at).floor if last_tick_at
+  end
+
+  def stopped?
+    heartbeat["stopped_at"].present?
+  end
+
+  # Ages are read on another machine's clock than they were written with, so a heartbeat
+  # from the future is believed only as far ahead as it would be believed old: a skewed
+  # clock cannot hold the lease for longer than a stale one would.
+  def alive?
+    limit = STALE_AFTER_INTERVALS * interval
+    !stopped? && !age.nil? && age.between?(-limit, limit)
+  end
+
+  def held_by_another?(id)
+    alive? && controller_id != id
+  end
+
+  def describe
+    tick = age ? "last tick #{age}s ago" : "no tick yet"
+    "controller #{controller_id} on #{heartbeat["hostname"]} (pid #{heartbeat["pid"]}, dash #{heartbeat["version"]}), #{tick}"
+  end
+end

data/lib/dash/autoscale/lease_held.rb

@@ -0,0 +1,2 @@
+# Another controller's heartbeat is alive, so this one does not start (`--takeover` overrides).
+class Dash::Autoscale::LeaseHeld < StandardError; end

data/lib/dash/autoscale/lease_lost.rb

@@ -0,0 +1,2 @@
+# The heartbeat names another controller - one started with `--takeover` - so this one stops.
+class Dash::Autoscale::LeaseLost < StandardError; end

data/lib/dash/autoscale/member.rb

@@ -0,0 +1,28 @@
+# A pool member as the provider reports it: its provider id, the address dash connects to,
+# the role it carries in its labels (all of them under `labels`), and its power state. `verified: false` marks a member
+# named with --hosts while the provider could not be asked (see Dash::Autoscale::Pool).
+Dash::Autoscale::Member = Struct.new(:id, :host, :role, :state, :labels, :created_at, :verified, keyword_init: true) do
+  def initialize(verified: true, **attributes)
+    super
+  end
+
+  def started?
+    state == "started"
+  end
+
+  def stopped?
+    state == "stopped"
+  end
+
+  # Neither on nor off: UpCloud reports `maintenance` while a server starts, stops or is
+  # changed, and `error` when it cannot.
+  def transitional?
+    !started? && !stopped?
+  end
+
+  def to_h
+    { id: id, host: host, role: role, state: state, labels: labels, created_at: created_at, verified: verified }
+  end
+end
+
+Dash::Autoscale::Member::STATES = %w[ started stopped maintenance error ].freeze

data/lib/dash/autoscale/metrics.rb

@@ -0,0 +1,70 @@
+# What `dash autoscale run --metrics-port` serves, in the Prometheus text format: per role
+# its running and target containers, its members per provider state, and its signal's and
+# guards' last samples as of the last tick, decision counts since the controller started,
+# and when the last tick ran and how long it took. Ticks write, the metrics server thread reads, so both go through the lock.
+class Dash::Autoscale::Metrics
+  CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
+
+  def initialize
+    @mutex = Mutex.new
+    @containers = {}
+    @decisions = Hash.new(0)
+    @members = {}
+    @signals = {}
+    @guards = {}
+    @last_tick_at = @tick_duration = nil
+  end
+
+  # `members` is { role => [ provider state of each member ] }.
+  def record_tick(decisions:, members:, at:, duration:)
+    @mutex.synchronize do
+      decisions.each do |decision|
+        @containers[decision.role] = { "running" => decision.from, "target" => decision.inputs[:floor] }.compact
+        decision.reasons.each { |reason| @decisions[[ decision.role, decision.action, reason ]] += 1 }
+        @signals[decision.role] = decision.inputs[:signal]
+        @guards[decision.role] = decision.inputs[:guards].to_a
+      end
+
+      members.each { |role, states| @members[role] = states.tally }
+      @last_tick_at, @tick_duration = at, duration
+    end
+  end
+
+  def render
+    @mutex.synchronize do
+      [
+        family("dash_autoscale_containers", "gauge", "Containers of a scaled role: running, and the target the controller holds it to.",
+          @containers.sort.flat_map { |role, counts| counts.sort.map { |state, count| [ { role: role, state: state }, count ] } }),
+        family("dash_autoscale_decisions_total", "counter", "Decisions the controller took, per role, action and reason.",
+          @decisions.sort.map { |(role, action, reason), count| [ { role: role, action: action, reason: reason }, count ] }),
+        family("dash_autoscale_members", "gauge", "Pool members of a scaled role, per provider state.",
+          @members.sort.flat_map { |role, states| states.sort.map { |state, count| [ { role: role, state: state }, count ] } }),
+        family("dash_autoscale_signal", "gauge", "The last sample of a scaled role's signal; absent while it cannot be read.",
+          @signals.sort.filter_map { |role, signal| [ { role: role, source: signal[:source] }, signal[:value] ] if signal&.dig(:value) }),
+        family("dash_autoscale_guard", "gauge", "The last sample of each of a scaled role's guards, by its index in hold_when.",
+          @guards.sort.flat_map { |role, guards| guards.filter_map { |guard| [ { role: role, guard: guard[:index], source: guard[:source] }, guard[:value] ] unless guard[:value].nil? } }),
+        family("dash_autoscale_last_tick_timestamp_seconds", "gauge", "When the last tick finished, in Unix seconds.",
+          @last_tick_at ? [ [ {}, @last_tick_at.to_i ] ] : []),
+        family("dash_autoscale_tick_duration_seconds", "gauge", "How long the last tick took.",
+          @tick_duration ? [ [ {}, @tick_duration.round(3) ] ] : [])
+      ].join
+    end
+  end
+
+  private
+    def family(name, type, help, samples)
+      lines = [ "# HELP #{name} #{help}", "# TYPE #{name} #{type}" ]
+      lines += samples.map { |labels, value| "#{name}#{labels(labels)} #{value}" }
+      lines.map { |line| "#{line}\n" }.join
+    end
+
+    def labels(labels)
+      return "" if labels.empty?
+
+      "{#{labels.map { |key, value| %(#{key}="#{escape(value)}") }.join(",")}}"
+    end
+
+    def escape(value)
+      value.to_s.gsub("\\", "\\\\\\\\").gsub("\"", "\\\"").gsub("\n", "\\n")
+    end
+end

data/lib/dash/autoscale/metrics_server.rb

@@ -0,0 +1,71 @@
+require "socket"
+
+# A one-thread HTTP endpoint for Prometheus: `GET /metrics` answers the controller's
+# Dash::Autoscale::Metrics, anything else is a 404. Binds 127.0.0.1 unless told otherwise
+# (`--metrics-bind`), so the metrics are not on the network by accident. No gem: a scrape
+# is one short request.
+class Dash::Autoscale::MetricsServer
+  READ_TIMEOUT = 5
+  MAX_HEADER_LINES = 100
+
+  def initialize(metrics, bind:, port:, timeout: READ_TIMEOUT)
+    @metrics, @bind, @port, @timeout = metrics, bind, port, timeout
+  end
+
+  def start
+    @server = TCPServer.new(@bind, @port)
+    @thread = Thread.new { serve }
+    self
+  end
+
+  def port
+    @server.addr[1]
+  end
+
+  def stop
+    @server&.close
+    @thread&.join(@timeout)
+  rescue IOError
+    nil
+  end
+
+  private
+    def serve
+      loop do
+        client = @server.accept
+        answer(client)
+      end
+    rescue IOError, Errno::EBADF
+      # Closed by #stop.
+    end
+
+    # One bad client never stops the endpoint: every read times out (IO::TimeoutError), and a
+    # request gets READ_TIMEOUT and MAX_HEADER_LINES in all, so a slow drip cannot hold it.
+    def answer(client)
+      client.timeout = @timeout
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout
+      method, path = client.gets.to_s.split(" ", 3)
+      # The headers are read too: closing a socket with them unread resets the connection,
+      # and a scraper can see the reset before the body.
+      MAX_HEADER_LINES.times do
+        break if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+        line = client.gets
+        break if line.nil? || line.strip.empty?
+      end
+
+      if method == "GET" && path == "/metrics"
+        respond client, "200 OK", @metrics.render
+      else
+        respond client, "404 Not Found", "Not found\n"
+      end
+    rescue StandardError
+      nil
+    ensure
+      client.close
+    end
+
+    def respond(client, status, body)
+      client.write "HTTP/1.1 #{status}\r\nContent-Type: #{Dash::Autoscale::Metrics::CONTENT_TYPE}\r\n" \
+        "Content-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+    end
+end

data/lib/dash/autoscale/observation.rb

@@ -0,0 +1,41 @@
+# What one scaled role looks like right now: how many of its containers run, which members
+# are started, which hosts did not answer, or the provider error when its pool could not be
+# read. One `docker ps` per host through Dash::Diagnostics::Scale, so one unreachable host
+# makes the role hold instead of failing the tick.
+#
+# A host that does not answer is either a started member (the controller may replace it
+# after `scale.boot_timeout`) or a baseline host (never touched: the role holds).
+class Dash::Autoscale::Observation
+  attr_reader :role, :current, :pool, :pool_error, :unreadable_baseline, :unreachable_members
+
+  def self.take(role)
+    status = Dash::Diagnostics::Scale.new(roles: [ role ]).to_h[:roles].first
+    return new(role: role, pool_error: status[:error]) if status[:error]
+
+    unread = status[:unread].to_h { |host| [ host[:host], host[:error] ] }
+    baseline = unread.slice(*role.baseline_hosts)
+
+    new(role: role, current: status[:total], pool: role.members, unreadable_baseline: baseline,
+      unreachable_members: unread.except(*baseline.keys))
+  end
+
+  # `pool` is every member of the role, whatever its state.
+  def initialize(role:, current: nil, pool: [], pool_error: nil, unreadable_baseline: {}, unreachable_members: {})
+    @role, @current, @pool, @pool_error = role, current, pool, pool_error
+    @unreadable_baseline, @unreachable_members = unreadable_baseline, unreachable_members
+  end
+
+  # The started members.
+  def members
+    pool.select(&:started?)
+  end
+
+  def member_hosts
+    members.map(&:host)
+  end
+
+  # Every host of the role that did not answer.
+  def unread_hosts
+    unreadable_baseline.keys + unreachable_members.keys
+  end
+end

data/lib/dash/autoscale/pause.rb

@@ -0,0 +1,34 @@
+# `dash autoscale pause ROLE [--for DURATION]`: the controller leaves the role alone until
+# `ends_at` (a Time) or for good (`:indefinite`) - `resume ROLE` removes it. Kept as
+# `pause/<role>.json` on the state host (Dash::Autoscale::StateStore.host), beside the
+# controller's other state.
+class Dash::Autoscale::Pause
+  INDEFINITE = "indefinite"
+
+  attr_reader :ends_at, :by, :at
+
+  def self.from(hash)
+    return unless hash.is_a?(Hash)
+
+    # A pause whose end cannot be read holds: letting go of a role an operator paused is
+    # the worse mistake.
+    ends_at = hash["until"] == INDEFINITE ? :indefinite : Dash::Autoscale::Timestamp.parse(hash["until"]) || :indefinite
+    new(ends_at: ends_at, by: hash["by"], at: Dash::Autoscale::Timestamp.parse(hash["at"]))
+  end
+
+  def initialize(ends_at:, by: nil, at: nil)
+    @ends_at, @by, @at = ends_at, by, at
+  end
+
+  def indefinite?
+    @ends_at == :indefinite
+  end
+
+  def active?(now)
+    indefinite? || now < @ends_at
+  end
+
+  def to_h
+    { "until" => indefinite? ? INDEFINITE : Dash::Autoscale::Timestamp.dump(@ends_at), "by" => by, "at" => Dash::Autoscale::Timestamp.dump(at) }
+  end
+end

data/lib/dash/autoscale/policy.rb

@@ -0,0 +1,198 @@
+# What the controller should do with one scaled role, now. Pure: every input is handed in -
+# the clock, the running count, the history of past actions, pauses, which members are
+# warming up or unreachable - and `decide` returns one Dash::Autoscale::Decision. No SSH, no
+# provider call, no clock of its own.
+#
+# The floor is scale min × replicas min, raised by any active schedule window, never above
+# scale max × replicas max. Below it the role scales out straight to it, whatever else holds.
+#
+# A role without a signal returns to its floor: above it, it scales in by `scale.step` at
+# most, once per `cooldown.down`, never while a member warms up. A role with a signal holds
+# it inside its band instead: `up.for` above `up.above` scales out by a step (unless a guard
+# holds, at max, or within `cooldown.up` of the last scale-out); `down.for` below
+# `down.below` scales in by a step, by the same rules as a return to the floor; inside the
+# band, or without a sample, it holds. Counts are containers; Dash::Cli::Scale::HostPlan
+# turns them into hosts.
+class Dash::Autoscale::Policy
+  attr_reader :role, :now, :current
+
+  # history:             { last_scale_out_at:, last_scale_in_at: }
+  # paused_until:        nil, a Time, or :indefinite
+  # warming:             { member host => joined at }
+  # unreachable:         { member host => unreachable since }
+  # unreadable_baseline: { baseline host => error (an exception or its message) }
+  # pool_error:          the provider's error (an exception or its message) when the pool could not be read
+  # signal:              this tick's Dash::Autoscale::Sample of the role's signal (a role with one)
+  # guards:              this tick's Dash::Autoscale::Guard::Readings
+  # breach:              { above_since:, below_since: } from Dash::Autoscale::RoleState#breach
+  def initialize(role:, now:, time_zone:, current:, history: {}, paused_until: nil, warming: {}, unreachable: {}, unreadable_baseline: {}, pool_error: nil,
+    signal: nil, guards: [], breach: {})
+    @role, @scale = role, role.scale
+    @now, @time_zone, @current = now, time_zone, current
+    @history, @paused_until, @warming, @unreachable = history, paused_until, warming, unreachable
+    @unreadable_baseline, @pool_error = unreadable_baseline, pool_error
+    @signal, @guards, @breach = signal, guards, breach
+  end
+
+  def decide
+    return hold("pool_unreadable", error: message(@pool_error)) if @pool_error
+    return hold("paused", eligible_at: (@paused_until unless @paused_until == :indefinite)) if paused?
+    return hold("host_unreachable", error: @unreadable_baseline.map { |host, error| "#{host}: #{message(error)}" }.join("; ")) if @unreadable_baseline.any?
+    return unreachable_member if @unreachable.any?
+
+    if current < floor
+      decision("scale_out", to: floor, reasons: floor_reasons)
+    elsif @scale.signal?
+      reactive
+    elsif current > floor
+      scale_in
+    else
+      hold("at_target")
+    end
+  end
+
+  def floor
+    @floor ||= [ [ @scale.min_count, *active_windows.map { |window| window[:min] } ].max, @scale.max_count ].min
+  end
+
+  private
+    def paused?
+      @paused_until == :indefinite || (@paused_until && now < @paused_until)
+    end
+
+    # One member per tick, the longest unreachable first, once it has had boot_timeout to
+    # answer and is past its warmup.
+    def unreachable_member
+      eligible_at = @unreachable.to_h { |host, since| [ host, [ since + @scale.boot_timeout, warmup_ends_at(host) ].compact.max ] }
+      ready = eligible_at.select { |_, at| now >= at }.keys
+
+      if ready.any?
+        decision("replace_member", to: current, reasons: [ "member_unreachable" ], replace: ready.min_by { |host| @unreachable[host] })
+      else
+        hold("member_unreachable", eligible_at: eligible_at.values.min)
+      end
+    end
+
+    def scale_in(reasons = floor_reasons, bottom: floor)
+      if warming_hosts.any?
+        hold("warming_up", eligible_at: warming_hosts.map { |host| warmup_ends_at(host) }.max)
+      elsif last_action_at && now < last_action_at + @scale.cooldown_down
+        hold("cooldown", eligible_at: last_action_at + @scale.cooldown_down)
+      else
+        decision("scale_in", to: step_target(bottom), reasons: reasons)
+      end
+    end
+
+    # At or above the floor, a signalled role follows its band - once it is within max: a
+    # role left above it (max lowered under its count) steps down to max first.
+    def reactive
+      return scale_in([ "at_max" ], bottom: @scale.max_count) if current > @scale.max_count
+      return hold("missing_data", error: "signal #{@scale.signal.source}: #{@signal&.error || "no value"}") if @signal.nil? || @signal.missing?
+
+      if (since = @breach[:above_since])
+        eligible_at = since + @scale.up_for
+        now >= eligible_at ? step_out : hold("above_threshold", eligible_at: eligible_at)
+      elsif (since = @breach[:below_since]) && current > floor
+        eligible_at = since + @scale.down_for
+        now >= eligible_at ? scale_in([ "below_threshold" ]) : hold("below_threshold", eligible_at: eligible_at)
+      else
+        hold("at_target")
+      end
+    end
+
+    def step_out
+      last_out = @history[:last_scale_out_at]
+
+      if (target = step_out_target).nil?
+        hold("at_max")
+      elsif (guard = @guards.find(&:holding?))
+        hold("held_by_guard", error: (guard.describe if guard.sample.missing?), held_by: guard.describe)
+      elsif last_out && now < last_out + @scale.cooldown_up
+        hold("cooldown", eligible_at: last_out + @scale.cooldown_up)
+      else
+        decision("scale_out", to: target, reasons: [ "above_threshold" ])
+      end
+    end
+
+    # A step up, never above max, landing on a count the hosts can split: the nearest one
+    # above, else the nearest one below that still adds a container. Nil at max.
+    def step_out_target
+      return if current >= @scale.max_count
+
+      target = [ current + @scale.step, @scale.max_count ].min
+
+      (target..@scale.max_count).find { |count| @scale.splittable?(count) } ||
+        target.downto(current + 1).find { |count| @scale.splittable?(count) }
+    end
+
+    # A step down, never below `bottom` (the floor), landing on a count the hosts can split:
+    # the nearest one above, else the nearest one below.
+    def step_target(bottom = floor)
+      target = [ bottom, current - @scale.step ].max
+
+      (target...current).find { |count| @scale.splittable?(count) } ||
+        target.downto(bottom).find { |count| @scale.splittable?(count) } || bottom
+    end
+
+    def floor_reasons
+      reasons = [ active_windows.any? { |window| window[:min] > @scale.min_count } ? "schedule_floor" : "at_min" ]
+      reasons << "at_max" if active_windows.any? { |window| window[:min] > @scale.max_count }
+      reasons
+    end
+
+    def active_windows
+      @active_windows ||= begin
+        local = now.in_time_zone(@time_zone)
+
+        @scale.schedule.filter_map do |window|
+          if (started_at = window.started_at(local))
+            window.to_h.merge(started_at: started_at.utc, ends_at: (started_at + window.duration).utc)
+          end
+        end
+      end
+    end
+
+    def warming_hosts
+      @warming_hosts ||= @warming.keys.select { |host| now < warmup_ends_at(host) }
+    end
+
+    def warmup_ends_at(host)
+      @warming[host]&.+(@scale.warmup)
+    end
+
+    def last_action_at
+      @history.values_at(:last_scale_out_at, :last_scale_in_at).compact.max
+    end
+
+    # Errors may come in as exceptions; a decision carries their message, which JSON can.
+    def message(error)
+      error.respond_to?(:message) ? error.message : error.to_s
+    end
+
+    def hold(reason, eligible_at: nil, error: nil, **extra)
+      decision("hold", to: current, reasons: [ reason ], eligible_at: eligible_at, error: error, **extra)
+    end
+
+    def decision(action, to:, reasons:, eligible_at: nil, error: nil, **extra)
+      Dash::Autoscale::Decision.new(role: role.name.to_s, action: action, from: current, to: to, reasons: reasons,
+        inputs: inputs.merge(extra), eligible_at: eligible_at, at: now, error: error)
+    end
+
+    def inputs
+      {
+        current: current, floor: floor, min_count: @scale.min_count, max_count: @scale.max_count, step: @scale.step,
+        cooldown_up: @scale.cooldown_up, cooldown_down: @scale.cooldown_down, warmup: @scale.warmup, boot_timeout: @scale.boot_timeout,
+        windows: active_windows, last_scale_out_at: @history[:last_scale_out_at], last_scale_in_at: @history[:last_scale_in_at],
+        warming: warming_hosts, unreachable: @unreachable, unreadable_baseline: @unreadable_baseline, paused_until: @paused_until
+      }.merge(signal_inputs)
+    end
+
+    # Everything Dash::Autoscale::Simulation needs to replay the tick, for a role with a signal.
+    def signal_inputs
+      return {} unless @scale.signal?
+
+      { signal: { source: @scale.signal.source, value: nil, at: nil, error: nil }.merge(@signal&.to_h || {}), guards: @guards.map(&:to_h),
+        above_since: @breach[:above_since], below_since: @breach[:below_since],
+        up_above: @scale.up_above, up_for: @scale.up_for, down_below: @scale.down_below, down_for: @scale.down_for }
+    end
+end