dash 4.1.3 → 4.2.0

data/lib/dash/autoscale/pool.rb

@@ -0,0 +1,103 @@
+# The pool members of every scaled role, as the provider reports them. The configuration
+# owns one and Role#hosts reads it, so members are asked for on the first `hosts` read and
+# then once per process per role - `refresh!` asks again.
+#
+# Degraded mode lives here and only here. When the provider cannot be asked and the command
+# named hosts with --hosts, the named hosts that are not baseline hosts are taken as
+# started members, unverified, of the one scaled role in scope (--roles narrows it), with a
+# warning. Without --hosts, or with more than one scaled role it could mean, the provider
+# error stands: dash never guesses which hosts are running.
+class Dash::Autoscale::Pool
+  attr_reader :config, :explicit_hosts, :explicit_roles
+
+  def initialize(config:, explicit_hosts: nil, explicit_roles: nil)
+    @config = config
+    @explicit_hosts = Array(explicit_hosts).presence
+    @explicit_roles = Array(explicit_roles).presence
+    @members = {}
+    @degraded = {}
+    @mutex = Mutex.new
+  end
+
+  # `on` runs host blocks in threads, and any of them may be the first to read a role's
+  # hosts; the lock keeps that to one provider call. A failure is kept too, so a provider
+  # that is down costs one timeout per process, not one per `hosts` read.
+  def members_for(role)
+    members = @mutex.synchronize do
+      @members[role.name] ||= begin
+        resolve(role)
+      rescue Dash::Autoscale::ProviderError => e
+        e
+      end
+    end
+
+    raise members if members.is_a?(Exception)
+    members
+  end
+
+  # Whether the role's members came from --hosts because the provider did not answer, so
+  # the role's full host list is unknown.
+  def degraded?(role)
+    members_for(role)
+    @degraded.fetch(role.name, false)
+  end
+
+  def active_for(role)
+    members_for(role).select(&:started?)
+  end
+
+  def refresh!
+    @mutex.synchronize { @members.clear; @degraded.clear }
+    self
+  end
+
+  def provider
+    @provider ||= Dash::Autoscale::Provider.for(config.autoscale)
+  end
+
+  def labels_for(role)
+    { "dash.service" => config.service, "dash.destination" => config.destination || "-", "dash.role" => role.name }
+  end
+
+  private
+    def resolve(role)
+      provider.members(labels: labels_for(role), address: role.scale.address)
+    # A credential that resolves to nothing only surfaces here, on the first request.
+    rescue Dash::Autoscale::ProviderError, Dash::ConfigurationError => e
+      degraded(role, e)
+    end
+
+    def degraded(role, error)
+      raise Dash::Autoscale::ProviderError, "Could not read the #{role} pool: #{error.message}" unless explicit_hosts
+
+      @degraded[role.name] = true
+
+      scoped = config.scaled_roles.select { |candidate| explicit_roles.nil? || Dash::Utils.filter_specific_items(explicit_roles, [ candidate ]).any? }
+      return [] unless scoped.include?(role)
+
+      if scoped.many?
+        raise Dash::Autoscale::ProviderError, "Could not read the pool (#{error.message}), " \
+          "and --hosts could mean members of #{scoped.map(&:name).join(" or ")}; name one with --roles"
+      end
+
+      unverified(role, error)
+    end
+
+    def unverified(role, error)
+      baseline = config.roles.flat_map(&:baseline_hosts) + config.accessories.flat_map(&:hosts)
+      # A wildcard names no host; only literal --hosts entries can stand in for members.
+      hosts = explicit_hosts.grep_v(/[*?\[{]/) - baseline
+
+      warn_once "Could not read the #{role} pool (#{error.message}); " \
+        "taking #{hosts.any? ? hosts.join(", ") : "no hosts"} from --hosts as started members, unverified"
+
+      hosts.map { |host| Dash::Autoscale::Member.new(id: nil, host: host, role: role.name, state: "started", labels: labels_for(role), verified: false) }
+    end
+
+    def warn_once(message)
+      return if @warned
+
+      @warned = true
+      warn message
+    end
+end

data/lib/dash/autoscale/prometheus_client.rb

@@ -0,0 +1,76 @@
+require "json"
+require "net/http"
+require "uri"
+
+# Prometheus's instant query API over Net::HTTP, as the gem's other HTTP client
+# (Dash::OtelShipper) uses it. A vector reads as its largest value, a scalar as its value;
+# no series is nil, and a NaN or infinite series NaN - both a missing sample. Through a
+# tunnel it connects to the tunnel (`ipaddr`) while TLS still names the URL's host.
+class Dash::Autoscale::PrometheusClient
+  class Error < StandardError; end
+
+  OPEN_TIMEOUT = 2
+  READ_TIMEOUT = 5
+
+  def initialize(url:, token: nil, tunnel: nil)
+    @uri = URI(url.end_with?("/") ? url : "#{url}/")
+    @token, @tunnel = token, tunnel
+  end
+
+  def query(promql)
+    uri = URI.join(@uri, "api/v1/query")
+    uri.query = URI.encode_www_form(query: promql)
+    request = Net::HTTP::Get.new(uri.request_uri)
+    request["Authorization"] = "Bearer #{@token}" if @token
+
+    value(data(http.request(request)))
+  end
+
+  # Nothing held open between queries.
+  def close; end
+
+  private
+    def http
+      Net::HTTP.new(@uri.host, @tunnel ? @tunnel.port : @uri.port).tap do |http|
+        http.ipaddr = @tunnel.address if @tunnel
+        http.use_ssl = @uri.scheme == "https"
+        http.open_timeout = OPEN_TIMEOUT
+        http.read_timeout = READ_TIMEOUT
+      end
+    end
+
+    def data(response)
+      body = begin
+        JSON.parse(response.body.to_s)
+      rescue JSON::ParserError
+        nil
+      end
+
+      return body["data"] if response.is_a?(Net::HTTPSuccess) && body.is_a?(Hash) && body["status"] == "success"
+
+      raise Error, "Prometheus answered HTTP #{response.code}#{": #{body["error"]}" if body.is_a?(Hash) && body["error"]}"
+    end
+
+    def value(data)
+      case data["resultType"]
+      when "vector"
+        values = data["result"].map { |series| number(series["value"][1]) }
+        return if values.empty?
+
+        values.all?(&:finite?) ? values.max : Float::NAN
+      when "scalar"
+        number(data["result"][1])
+      else
+        raise Error, "Prometheus returned a #{data["resultType"]}, not a vector or a scalar: make the query return one number"
+      end
+    end
+
+    def number(text)
+      case text
+      when "NaN" then Float::NAN
+      when "+Inf" then Float::INFINITY
+      when "-Inf" then -Float::INFINITY
+      else Float(text)
+      end
+    end
+end

data/lib/dash/autoscale/provider/base.rb

@@ -0,0 +1,43 @@
+# What every provider answers. `labels` is the role's membership:
+# { "dash.service" => ..., "dash.destination" => ..., "dash.role" => ... }.
+#
+#   members(labels:, address:)              -> [ Member ] carrying exactly those labels
+#   start(member)                           -> powers it on
+#   stop(member, timeout:)                  -> powers it off, soft, within timeout seconds
+#   create(labels:, template:, address:)    -> Member, the new server
+#   destroy(member)                         -> deletes it and its storage
+#   state(member)                           -> "started" | "stopped" | "maintenance" | "error"
+#
+# Every failure raises Dash::Autoscale::ProviderError.
+class Dash::Autoscale::Provider::Base
+  POLL_INTERVAL = 5
+
+  def name
+    self.class.name.demodulize.downcase
+  end
+
+  %i[ members start stop create destroy state ].each do |action|
+    define_method(action) { |*, **| raise NotImplementedError, "#{self.class.name}##{action}" }
+  end
+
+  # Polls until the member reaches `state`, or raises once `timeout` seconds have passed.
+  def wait_until(member, state:, timeout:, interval: POLL_INTERVAL)
+    deadline = monotonic_now + timeout
+
+    loop do
+      current = state(member)
+      return current if current == state
+
+      if monotonic_now >= deadline
+        raise Dash::Autoscale::ProviderError, "#{name}: #{member.id} is #{current}, not #{state}, after #{timeout}s"
+      end
+
+      sleep interval
+    end
+  end
+
+  private
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+end

data/lib/dash/autoscale/provider/exec.rb

@@ -0,0 +1,94 @@
+require "json"
+require "open3"
+
+# The operator's own scripts, for any cloud dash has no client for. Each runs locally with
+# the role's labels as DASH_SERVICE / DASH_DESTINATION / DASH_ROLE (and their KAMAL_
+# twins, as hooks get), plus DASH_MEMBER_ID and DASH_HOST for the member it acts on.
+# `members` and `create` print members as JSON, one object per line.
+class Dash::Autoscale::Provider::Exec < Dash::Autoscale::Provider::Base
+  def initialize(scripts:)
+    @scripts = scripts
+  end
+
+  # `address` is the script's business: it prints the host dash should connect to.
+  def members(labels:, address:)
+    run(:members, labels).lines.filter_map { |line| parse_member(line, labels) if line.strip.present? }
+      .select { |member| member.role == labels["dash.role"] }
+  end
+
+  def start(member)
+    run(:start, labels_of(member), member)
+  end
+
+  def stop(member, timeout:)
+    run(:stop, labels_of(member), member, "DASH_STOP_TIMEOUT" => timeout.to_s)
+  end
+
+  def create(labels:, template:, address:)
+    parse_member(run(:create, labels).lines.find { |line| line.strip.present? }, labels) ||
+      raise(Dash::Autoscale::ProviderError, "exec: create printed no member")
+  end
+
+  def destroy(member)
+    run(:destroy, labels_of(member), member)
+  end
+
+  # The members script is the only source of state there is.
+  def state(member)
+    members(labels: labels_of(member), address: nil).find { |candidate| candidate.id == member.id }&.state ||
+      raise(Dash::Autoscale::ProviderError, "exec: members no longer lists #{member.id}")
+  end
+
+  private
+    def run(action, labels, member = nil, extra_env = {})
+      script = @scripts.call(action) || raise(Dash::Autoscale::ProviderError, "exec: no #{action} script, set autoscale/provider/exec/#{action}")
+      output, error, status = Open3.capture3(env(labels, member).merge(extra_env), script)
+
+      unless status.success?
+        raise Dash::Autoscale::ProviderError, "exec: #{script} exited #{status.exitstatus}#{": #{last_line(error)}" if error.present?}"
+      end
+
+      output
+    rescue SystemCallError => e
+      raise Dash::Autoscale::ProviderError, "exec: #{script} could not run (#{e.message})"
+    end
+
+    def env(labels, member)
+      Dash::Tags.new(
+        service: labels["dash.service"], destination: labels["dash.destination"], role: labels["dash.role"],
+        member_id: member&.id, host: member&.host
+      ).env
+    end
+
+    def parse_member(line, labels)
+      return if line.nil?
+
+      data = JSON.parse(line)
+      raise JSON::ParserError unless data.is_a?(Hash)
+
+      if data["id"].to_s.blank? || data["host"].to_s.blank?
+        raise Dash::Autoscale::ProviderError, "exec: a member needs an id and a host, got #{line.strip.truncate(80).inspect}"
+      end
+
+      state = data["state"].to_s
+
+      unless Dash::Autoscale::Member::STATES.include?(state)
+        raise Dash::Autoscale::ProviderError, "exec: member #{data["id"]} has state #{state.inspect}, expected one of #{Dash::Autoscale::Member::STATES.join(", ")}"
+      end
+
+      Dash::Autoscale::Member.new(id: data["id"].to_s, host: data["host"], role: data["role"] || labels["dash.role"], state: state,
+        labels: labels.merge("dash.role" => data["role"] || labels["dash.role"]), created_at: data["created_at"])
+    rescue JSON::ParserError
+      raise Dash::Autoscale::ProviderError, "exec: expected one JSON object per line, got #{line.strip.truncate(80).inspect}"
+    end
+
+    # The last line says why a script failed; anything above it is the script's own chatter,
+    # and the message ends up in errors and saved reports.
+    def last_line(output)
+      output.lines.map(&:strip).reject(&:blank?).last.to_s.truncate(200)
+    end
+
+    def labels_of(member)
+      member.labels || raise(Dash::Autoscale::ProviderError, "exec: member #{member.id} carries no labels")
+    end
+end

data/lib/dash/autoscale/provider/upcloud.rb

@@ -0,0 +1,140 @@
+require "net/http"
+require "json"
+require "securerandom"
+require "uri"
+
+# UpCloud API 1.3 (https://developers.upcloud.com/1.3/8-servers/) over Net::HTTP. The
+# credentials are lambdas so the secrets are read on the first request, never earlier, and
+# they only ever travel as basic auth - error messages carry the path and the status.
+class Dash::Autoscale::Provider::Upcloud < Dash::Autoscale::Provider::Base
+  API = URI("https://api.upcloud.com")
+  OPEN_TIMEOUT = 10
+  READ_TIMEOUT = 60
+
+  NETWORK_ERRORS = [ SocketError, SystemCallError, IOError, Timeout::Error, OpenSSL::SSL::SSLError, Net::ProtocolError ]
+
+  def initialize(username:, password:)
+    @username = username
+    @password = password
+  end
+
+  # The list is filtered by label on the server and again here: a member is only ever a
+  # server that carries every one of the role's labels. The list has no addresses, so each
+  # match is read once more for them.
+  def members(labels:, address:)
+    query = URI.encode_www_form(labels.map { |key, value| [ "label", "#{key}=#{value}" ] })
+    servers = request(:get, "/1.3/server?#{query}").dig("servers", "server") || []
+
+    servers.select { |server| labelled?(server, labels) }.map do |server|
+      member_from(request(:get, "/1.3/server/#{server["uuid"]}").fetch("server"), labels: labels, address: address)
+    end
+  end
+
+  def start(member)
+    request(:post, "/1.3/server/#{member.id}/start", {})
+  end
+
+  def stop(member, timeout:)
+    request(:post, "/1.3/server/#{member.id}/stop", { stop_server: { stop_type: "soft", timeout: timeout.to_s } })
+  end
+
+  def create(labels:, template:, address:)
+    server = request(:post, "/1.3/server", { server: create_body(labels: labels, template: template) }).fetch("server")
+    member_from(server, labels: labels, address: address)
+  end
+
+  def destroy(member)
+    request(:delete, "/1.3/server/#{member.id}?storages=1")
+  end
+
+  def state(member)
+    request(:get, "/1.3/server/#{member.id}").dig("server", "state")
+  end
+
+  private
+    def labelled?(server, labels)
+      present = Array(server.dig("labels", "label")).to_h { |label| [ label["key"], label["value"] ] }
+      labels.all? { |key, value| present[key] == value }
+    end
+
+    def member_from(server, labels:, address:)
+      Dash::Autoscale::Member.new \
+        id: server["uuid"],
+        host: address_of(server, address) || raise(Dash::Autoscale::ProviderError, "upcloud: server #{server["uuid"]} has no #{address} IPv4 address, set scale/address to one it has"),
+        role: labels["dash.role"],
+        state: server["state"],
+        labels: labels,
+        created_at: server["created"]
+    end
+
+    # IPv4 only: an IPv6 host would need brackets everywhere dash builds `host:port`.
+    def address_of(server, access)
+      Array(server.dig("ip_addresses", "ip_address")).find { |ip| ip["access"] == access && ip["family"] == "IPv4" }&.fetch("address")
+    end
+
+    # Public and utility interfaces always, so a member can pull its image and reach the
+    # zone's private services; the SDN interface when the template names one.
+    def create_body(labels:, template:)
+      name = [ labels["dash.service"], labels["dash.role"], SecureRandom.hex(4) ].join("-")
+
+      {
+        zone: template["zone"],
+        title: name,
+        hostname: name,
+        plan: template["plan"],
+        metadata: "yes",
+        labels: { label: labels.map { |key, value| { key: key, value: value } } },
+        storage_devices: { storage_device: [ { action: "clone", storage: template["storage"], title: "#{name}-disk" } ] },
+        networking: { interfaces: { interface: interfaces(template["network"]) } },
+        login_user: login_user(template),
+        user_data: template["user_data"]
+      }.compact
+    end
+
+    def interfaces(network)
+      ipv4 = { ip_address: [ { family: "IPv4" } ] }
+
+      [ { index: 1, type: "public", ip_addresses: ipv4 }, { index: 2, type: "utility", ip_addresses: ipv4 } ].tap do |interfaces|
+        interfaces << { index: 3, type: "private", network: network, ip_addresses: ipv4 } if network.present?
+      end
+    end
+
+    def login_user(template)
+      if template["login_user"].present?
+        { username: template["login_user"], create_password: "no", ssh_keys: { ssh_key: Array(template["ssh_keys"]) } }
+      end
+    end
+
+    def request(method, path, body = nil)
+      request = Net::HTTP.const_get(method.to_s.capitalize).new(path, "Accept" => "application/json")
+      request.basic_auth(@username.call, @password.call)
+
+      if body
+        request["Content-Type"] = "application/json"
+        request.body = JSON.generate(body)
+      end
+
+      response = nil
+      Net::HTTP.start(API.host, API.port, use_ssl: true, open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT) { |http| response = http.request(request) }
+      parse(method, path, response)
+    rescue *NETWORK_ERRORS => e
+      raise Dash::Autoscale::ProviderError, "upcloud: #{method.upcase} #{path.split("?").first} failed (#{e.class}: #{e.message})"
+    end
+
+    def parse(method, path, response)
+      unless response.is_a?(Net::HTTPSuccess)
+        raise Dash::Autoscale::ProviderError, "upcloud: #{method.upcase} #{path.split("?").first} answered #{response.code} #{api_error(response)}".strip
+      end
+
+      response.body.present? ? JSON.parse(response.body) : {}
+    rescue JSON::ParserError
+      raise Dash::Autoscale::ProviderError, "upcloud: #{method.upcase} #{path.split("?").first} answered something that is not JSON"
+    end
+
+    def api_error(response)
+      error = JSON.parse(response.body.to_s).fetch("error", {})
+      [ error["error_code"], error["error_message"] ].compact.join(": ")
+    rescue JSON::ParserError
+      response.message
+    end
+end

data/lib/dash/autoscale/provider.rb

@@ -0,0 +1,9 @@
+module Dash::Autoscale::Provider
+  def self.for(autoscale)
+    case autoscale.provider_name
+    when "upcloud" then Dash::Autoscale::Provider::Upcloud.new(username: -> { autoscale.upcloud_username }, password: -> { autoscale.upcloud_password })
+    when "exec" then Dash::Autoscale::Provider::Exec.new(scripts: ->(action) { autoscale.exec_script(action) })
+    else raise Dash::Autoscale::ProviderError, "no autoscale provider configured"
+    end
+  end
+end

data/lib/dash/autoscale/provider_error.rb

@@ -0,0 +1,4 @@
+# The provider could not be asked, or refused. dash never puts a credential in the message:
+# UpCloud errors carry the request path and the response status, exec errors the script
+# name and the last line of its stderr - which is the operator's script to keep clean.
+class Dash::Autoscale::ProviderError < StandardError; end

data/lib/dash/autoscale/read_only_query.rb

@@ -0,0 +1,38 @@
+require "timeout"
+
+# How the controller reads Postgres: inside a read-only transaction with a statement
+# timeout, always rolled back, so a signal or guard query can neither write nor hang a tick.
+# The statement timeout bounds the server; DEADLINE bounds the client, for a connection that
+# stops answering at all (pg's `exec` waits on the socket in Ruby, so it can be interrupted).
+module Dash::Autoscale::ReadOnlyQuery
+  STATEMENT_TIMEOUT_MS = 5000
+  DEADLINE = 10
+
+  # The connection stopped answering. A Timeout::Error, so Dash::Autoscale::Connections
+  # discards the connection - which ends its transaction on the server.
+  class Stalled < Timeout::Error; end
+
+  def self.within(connection, deadline: DEADLINE)
+    Timeout.timeout(deadline, Stalled) do
+      connection.exec("BEGIN READ ONLY")
+      stalled = false
+      begin
+        connection.exec("SET LOCAL statement_timeout = #{STATEMENT_TIMEOUT_MS}")
+        yield
+      rescue Stalled
+        stalled = true
+        raise
+      ensure
+        # A ROLLBACK on a stalled socket would wait on it again, past the deadline.
+        connection.exec("ROLLBACK") unless stalled
+      end
+    end
+  end
+
+  # The first column of the first row as a number; nil without a row or for SQL NULL.
+  def self.number(connection, sql)
+    result = connection.exec(sql)
+    value = result.getvalue(0, 0) unless result.ntuples.zero?
+    Float(value) unless value.nil?
+  end
+end

data/lib/dash/autoscale/reader.rb

@@ -0,0 +1,17 @@
+# Reads a signalled role's signal and guards for one evaluation, through the process's
+# Dash::Autoscale::Connections. The controller hands Dash::Autoscale::Evaluation one; tests
+# hand it a fake.
+class Dash::Autoscale::Reader
+  def initialize(connections = Dash::Autoscale::Connections.shared)
+    @connections = connections
+  end
+
+  # [ the signal's Sample, the guards' Dash::Autoscale::Guard::Readings ]
+  def read(role, at:)
+    [ Dash::Autoscale::Signal.for(role, @connections).sample(at: at), Dash::Autoscale::Guard.read(role.scale.guards, @connections, at: at) ]
+  end
+
+  def close
+    @connections.close
+  end
+end

data/lib/dash/autoscale/recorder.rb

@@ -0,0 +1,13 @@
+require "json"
+
+# `dash autoscale run --record FILE`: every tick's decisions, with every policy input they
+# were made from, appended to a local JSONL file - one line per tick - for replaying later.
+class Dash::Autoscale::Recorder
+  def initialize(path)
+    @path = path
+  end
+
+  def record(at, decisions)
+    File.open(@path, "a") { |file| file.puts JSON.generate(at: Dash::Autoscale::Timestamp.dump(at), decisions: decisions.map(&:to_h)) }
+  end
+end

data/lib/dash/autoscale/role_state.rb

@@ -0,0 +1,110 @@
+# What the controller remembers about one scaled role between ticks, in `state.json` on
+# the state host so a restart or a `--once` run from cron carries on where the last tick
+# stopped: when it last scaled out and in (cooldown), when each member it joined came up
+# (warmup), since when each started member has not answered (replacement after
+# `boot_timeout`), since when its signal has been above or below its band (`up.for`,
+# `down.for`), and the last decision it logged (the log only takes changes).
+#
+# The bridge between the file (string keys, ISO 8601 strings) and Dash::Autoscale::Policy
+# (symbols, Time).
+class Dash::Autoscale::RoleState
+  attr_reader :unreachable, :joined, :last_decision
+
+  def self.from(hash)
+    hash = hash.is_a?(Hash) ? hash : {}
+
+    new(last_scale_out_at: Dash::Autoscale::Timestamp.parse(hash["last_scale_out_at"]),
+      last_scale_in_at: Dash::Autoscale::Timestamp.parse(hash["last_scale_in_at"]),
+      unreachable: times(hash["unreachable"]), joined: times(hash["joined"]),
+      last_decision: hash["last_decision"].is_a?(Hash) ? hash["last_decision"] : nil,
+      above_since: Dash::Autoscale::Timestamp.parse(hash["above_since"]), below_since: Dash::Autoscale::Timestamp.parse(hash["below_since"]))
+  end
+
+  def self.times(hash)
+    return {} unless hash.is_a?(Hash)
+
+    hash.filter_map { |host, time| [ host, Dash::Autoscale::Timestamp.parse(time) ] if Dash::Autoscale::Timestamp.parse(time) }.to_h
+  end
+
+  def initialize(last_scale_out_at: nil, last_scale_in_at: nil, unreachable: {}, joined: {}, last_decision: nil, above_since: nil, below_since: nil)
+    @last_scale_out_at, @last_scale_in_at = last_scale_out_at, last_scale_in_at
+    @unreachable, @joined, @last_decision = unreachable, joined, last_decision
+    @above_since, @below_since = above_since, below_since
+  end
+
+  def history
+    { last_scale_out_at: @last_scale_out_at, last_scale_in_at: @last_scale_in_at }
+  end
+
+  def breach
+    { above_since: @above_since, below_since: @below_since }
+  end
+
+  # Every sample moves the breach windows before the policy reads them: above the band the
+  # above window starts (or keeps its start), below it the below window does, and inside
+  # the band - or without a number to compare - both end. A gap in samples restarts a window.
+  def observe_signal(sample, up_above:, down_below:)
+    if sample.nil? || sample.missing?
+      @above_since = @below_since = nil
+    elsif sample.value > up_above
+      @above_since ||= sample.at
+      @below_since = nil
+    elsif sample.value < down_below
+      @below_since ||= sample.at
+      @above_since = nil
+    else
+      @above_since = @below_since = nil
+    end
+  end
+
+  # The members that did not answer this tick: each keeps the time it was first seen
+  # unreachable, a new one starts now, and one that answered again is forgotten.
+  def observe_unreachable(hosts, now:)
+    @unreachable = hosts.to_h { |host| [ host, @unreachable.fetch(host, now) ] }
+  end
+
+  def joined_at(hosts, now:)
+    hosts.each { |host| @joined[host] = now }
+  end
+
+  # Only the members still started can be warming up.
+  def keep_members(hosts)
+    @joined = @joined.slice(*hosts)
+  end
+
+  def scaled(action, now:)
+    case action
+    when "scale_out" then @last_scale_out_at = now
+    when "scale_in" then @last_scale_in_at = now
+    end
+  end
+
+  def forget(host)
+    @unreachable = @unreachable.except(host)
+    @joined = @joined.except(host)
+  end
+
+  # Every action is logged; a hold only when its action or reasons differ from the last
+  # decision logged, so a steady role adds nothing to the log tick after tick.
+  def log?(decision)
+    !decision.hold? || last_decision != logged_shape(decision)
+  end
+
+  def logged(decision)
+    @last_decision = logged_shape(decision)
+  end
+
+  def to_h
+    { "last_scale_out_at" => Dash::Autoscale::Timestamp.dump(@last_scale_out_at),
+      "last_scale_in_at" => Dash::Autoscale::Timestamp.dump(@last_scale_in_at),
+      "unreachable" => @unreachable.transform_values { |time| Dash::Autoscale::Timestamp.dump(time) },
+      "joined" => @joined.transform_values { |time| Dash::Autoscale::Timestamp.dump(time) },
+      "last_decision" => @last_decision }
+      .merge({ "above_since" => Dash::Autoscale::Timestamp.dump(@above_since), "below_since" => Dash::Autoscale::Timestamp.dump(@below_since) }.compact)
+  end
+
+  private
+    def logged_shape(decision)
+      { "action" => decision.action, "reasons" => decision.reasons }
+    end
+end

data/lib/dash/autoscale/sample.rb

@@ -0,0 +1,13 @@
+# One reading of a signal or a guard: the number, when it was read, or the error that kept
+# it from being read. A reader never raises out of a tick - a failure is a Sample with an
+# error, and the policy holds the role with `missing_data`.
+Dash::Autoscale::Sample = Struct.new(:value, :at, :error, keyword_init: true) do
+  # No number to act on: an error, nothing read, or a NaN/Inf a query returned.
+  def missing?
+    !error.nil? || !value.is_a?(Numeric) || (value.is_a?(Float) && !value.finite?)
+  end
+
+  def to_h
+    { value: (value unless missing?), at: at, error: error.respond_to?(:message) ? error.message : error }
+  end
+end