dash 4.1.3 → 4.2.0

data/lib/dash/autoscale/signal/pgbus.rb

@@ -0,0 +1,19 @@
+# The age in seconds of the oldest claimable message across the role's queues. A pgbus queue
+# `<name>` is the pgmq table `pgmq.q_<prefix><name>`; a message is claimable once its
+# visibility time `vt` has passed, so scheduled messages do not count (pgmq.metrics()'s
+# oldest_msg_age_sec does count them). pgbus reports no jobs in progress, so a pgbus role
+# drains on `docker stop` alone.
+class Dash::Autoscale::Signal::Pgbus < Dash::Autoscale::Signal
+  private
+    def read(_at)
+      connections.with(:postgres) do |postgres|
+        Dash::Autoscale::ReadOnlyQuery.within(postgres) do
+          config.queues.map { |queue| Dash::Autoscale::ReadOnlyQuery.number(postgres, claimable_age(postgres, queue)) || 0 }.max
+        end
+      end
+    end
+
+    def claimable_age(postgres, queue)
+      "SELECT EXTRACT(epoch FROM now() - min(vt)) FROM pgmq.#{postgres.quote_ident("q_#{config.prefix}#{queue}")} WHERE vt <= now()"
+    end
+end

data/lib/dash/autoscale/signal/prometheus.rb

@@ -0,0 +1,8 @@
+# The value of one PromQL query, the largest when it returns several series. No series, NaN
+# or Inf is a missing sample (Dash::Autoscale::Sample#missing?).
+class Dash::Autoscale::Signal::Prometheus < Dash::Autoscale::Signal
+  private
+    def read(_at)
+      connections.with(:prometheus) { |prometheus| prometheus.query(config.query) }
+    end
+end

data/lib/dash/autoscale/signal/sidekiq.rb

@@ -0,0 +1,49 @@
+require "json"
+
+# The age in seconds of the oldest job waiting across the role's queues. A Sidekiq queue is
+# the Redis list `queue:<name>`, pushed on the left and fetched from the right, so its
+# oldest job is `LINDEX queue:<name> -1`. Sidekiq 7 and older write `enqueued_at` as float
+# epoch seconds, Sidekiq 8 as integer epoch milliseconds. Reads only: LINDEX, SMEMBERS, HMGET.
+class Dash::Autoscale::Signal::Sidekiq < Dash::Autoscale::Signal
+  # Epoch seconds pass this in the year 5138; epoch milliseconds passed it in 1973.
+  MILLISECONDS_ABOVE = 1e11
+
+  def busy?
+    true
+  end
+
+  # Jobs in progress in the containers whose hostnames are given, or nil unless Sidekiq knows
+  # a process in every one of them: an idle process elsewhere does not speak for a container
+  # Sidekiq has no record of. Sidekiq reports each process's `hostname` (the container's) in its
+  # `info`, and writes `busy` and `quiet` on every heartbeat (every few seconds) - so a busy
+  # count from before the process went quiet may predate the drain signal and miss a job it
+  # fetched since. Until a process reports quiet it counts as at least one job busy.
+  def busy(hostnames)
+    connections.with(:redis) do |redis|
+      counts = Array(redis.call("SMEMBERS", "processes")).filter_map do |identity|
+        info, busy, quiet = redis.call("HMGET", identity, "info", "busy", "quiet")
+        hostname = JSON.parse(info)["hostname"] if info
+        next unless hostnames.include?(hostname)
+
+        [ hostname, quiet == "true" ? busy.to_i : [ busy.to_i, 1 ].max ]
+      end
+
+      counts.sum(&:last) if (hostnames - counts.map(&:first)).empty?
+    end
+  end
+
+  private
+    def read(at)
+      connections.with(:redis) do |redis|
+        config.queues.map { |queue| age(redis.call("LINDEX", "queue:#{queue}", -1), at) }.max
+      end
+    end
+
+    def age(job, at)
+      return 0 if job.nil?
+
+      enqueued_at = JSON.parse(job).fetch("enqueued_at")
+      enqueued_at /= 1000.0 if enqueued_at > MILLISECONDS_ABOVE
+      [ at.to_f - enqueued_at, 0 ].max
+    end
+end

data/lib/dash/autoscale/signal.rb

@@ -0,0 +1,31 @@
+# A role's `scale.signal` as the controller reads it: one number per tick, from Redis
+# (Sidekiq), Postgres (pgbus) or Prometheus, through the process's
+# Dash::Autoscale::Connections. A reader never raises out of a tick: a failure becomes a
+# Sample with its error, and the policy holds the role. Only a configuration error (a
+# missing gem, a secret that resolves to nothing) goes through, for the operator to fix.
+class Dash::Autoscale::Signal
+  attr_reader :config, :connections
+
+  def self.for(role, connections)
+    { "sidekiq" => Sidekiq, "pgbus" => Pgbus, "prometheus" => Prometheus }.fetch(role.scale.signal.source).new(role.scale.signal, connections)
+  end
+
+  # `config` is the role's Dash::Configuration::Role::Scale::Signal.
+  def initialize(config, connections)
+    @config, @connections = config, connections
+  end
+
+  def sample(at: Time.now.utc)
+    Dash::Autoscale::Sample.new(value: read(at), at: at)
+  rescue Dash::ConfigurationError
+    raise
+  rescue StandardError => e
+    Dash::Autoscale::Sample.new(value: nil, at: at, error: "#{e.class}: #{e.message}")
+  end
+
+  # Whether the source can say how many jobs a container is working on (only Sidekiq can):
+  # a drain then ends as soon as it says none.
+  def busy?
+    false
+  end
+end

data/lib/dash/autoscale/simulation.rb

@@ -0,0 +1,200 @@
+require "json"
+
+# `dash autoscale simulate --samples FILE`: replays a `dash autoscale run --record` file
+# through Dash::Autoscale::Policy with the thresholds in deploy.yml now, to try a band, a
+# step or a cooldown against a day that happened before deploying it.
+#
+# Each role starts at the count its first tick recorded. Every tick hands the policy that
+# tick's recorded signal and guard readings with the simulated count and a simulated
+# RoleState: the breach windows and the last scale times are worked out again with the
+# current thresholds (the recorded ones were measured against the old band), and a
+# simulated action changes the count at once. A scale-out onto more hosts warms them up for
+# `scale.warmup`, as joined members do. The world's own holds are replayed: a recorded
+# pause, an unreadable pool, unreadable baseline hosts, unreachable members (a replacement
+# the policy then decides keeps the count, as it did live).
+#
+# Known limits, so a replay with the same thresholds can differ from what ran:
+# - the recorded signal is what the queue did with the containers that actually ran, not
+#   with the simulated ones;
+# - a simulated action's cooldown and warmup start at its tick, where live they start once
+#   `dash scale set` returns (a join can take minutes), so the simulation can act again a
+#   tick or more early;
+# - the breach windows start empty, so a recording that begins inside a breach scales
+#   `up.for` (or `down.for`) later than live did.
+class Dash::Autoscale::Simulation
+  ACTIONS = %w[ scale_out scale_in ].freeze
+
+  # `roles`: role names to replay, or nil for every role in the file.
+  def initialize(path:, config:, roles: nil)
+    @path, @config, @roles = path, config, roles
+  end
+
+  def run
+    ticks, errors = read
+    replays, skipped = [], []
+
+    ticks.group_by { |_, decision| decision["role"] }.each do |name, role_ticks|
+      next if @roles && !@roles.include?(name)
+
+      role = @config.role(name)
+      if role&.scaled? && role.controlled?
+        replays << Replay.new(role, role_ticks).to_h
+      else
+        skipped << { role: name, reason: role ? "#{name} has no scale/schedule or scale/signal in deploy.yml" : "no role named #{name} in deploy.yml" }
+      end
+    end
+
+    { file: @path, roles: replays, skipped: skipped, errors: errors }
+  end
+
+  private
+    # [ [ at, recorded decision ] in file order, [ { line:, error: } ] ]
+    def read
+      ticks, errors = [], []
+
+      File.foreach(@path).with_index(1) do |line, number|
+        next if line.strip.empty?
+
+        tick = JSON.parse(line)
+        at = Dash::Autoscale::Timestamp.parse(tick["at"]) if tick.is_a?(Hash)
+        decisions = tick["decisions"] if tick.is_a?(Hash)
+        unless at && decisions.is_a?(Array) && decisions.all? { |decision| decision.is_a?(Hash) && decision["role"].is_a?(String) }
+          raise ArgumentError, "line is not a recorded tick: it needs an at time and a list of decisions, each naming its role"
+        end
+
+        decisions.each { |decision| ticks << [ at, decision ] }
+      rescue JSON::ParserError, ArgumentError => e
+        errors << { line: number, error: e.message }
+      end
+
+      [ ticks, errors ]
+    end
+
+    # One role's replay over its recorded ticks.
+    class Replay
+      def initialize(role, ticks)
+        @role, @scale, @ticks = role, role.scale, ticks.sort_by(&:first)
+        @state = Dash::Autoscale::RoleState.new
+        @actions, @counts, @not_replayable, @joins = [], [], 0, 0
+      end
+
+      def to_h
+        replay
+
+        { role: @role.name, ticks: @ticks.size, replayed: @ticks.size - @not_replayable, not_replayable: @not_replayable,
+          unrecorded_guards: unrecorded_guards, actions: @actions.map { |decision| decision.to_h.merge(summary: decision.summary) },
+          simulated: summary(@actions.count { |decision| decision.action == "scale_out" }, @actions.count { |decision| decision.action == "scale_in" }, @counts),
+          recorded: summary(*recorded_actions, recorded_counts) }.compact
+      end
+
+      private
+        def replay
+          @ticks.each do |at, recorded|
+            inputs = recorded["inputs"].is_a?(Hash) ? recorded["inputs"] : {}
+            seed(recorded, inputs) if @count.nil?
+
+            if @count.nil? || (@scale.signal? && !inputs["signal"].is_a?(Hash))
+              @not_replayable += 1
+            else
+              act(at, decide(at, recorded, inputs))
+            end
+
+            @counts << [ at, @count ]
+          end
+        end
+
+        # The replay starts at the first tick that recorded a count: its count, and the last
+        # scale times it recorded, so cooldowns hold from there.
+        def seed(recorded, inputs)
+          return unless recorded["from"].is_a?(Integer)
+
+          @count = recorded["from"]
+          @state = Dash::Autoscale::RoleState.new(last_scale_out_at: Dash::Autoscale::Timestamp.parse(inputs["last_scale_out_at"]),
+            last_scale_in_at: Dash::Autoscale::Timestamp.parse(inputs["last_scale_in_at"]))
+        end
+
+        def decide(at, recorded, inputs)
+          signal = sample(inputs["signal"], at) if @scale.signal?
+          @state.observe_signal(signal, up_above: @scale.up_above, down_below: @scale.down_below) if signal
+
+          Dash::Autoscale::Policy.new(role: @role, now: at, time_zone: @role.config.autoscale.time_zone, current: @count,
+            history: @state.history, paused_until: paused_until(inputs["paused_until"]), warming: @state.joined, unreachable: unreachable(inputs["unreachable"]),
+            unreadable_baseline: inputs["unreadable_baseline"].is_a?(Hash) ? inputs["unreadable_baseline"] : {},
+            pool_error: (recorded["error"] if recorded["reasons"] == [ "pool_unreadable" ]),
+            signal: signal, guards: (guards(inputs["guards"], at) if @scale.signal?).to_a, breach: @state.breach).decide
+        end
+
+        def act(at, decision)
+          return unless ACTIONS.include?(decision.action)
+
+          new_hosts = hosts(decision.to) - hosts(@count)
+          @state.scaled(decision.action, now: at)
+          @state.joined_at(new_hosts.times.map { "simulated-#{@joins += 1}" }, now: at) if decision.action == "scale_out" && new_hosts.positive?
+          @count = decision.to
+          @actions << decision
+        end
+
+        # As Dash::Configuration::Role::Scale#splittable? reckons it.
+        def hosts(count)
+          [ (count.to_f / @scale.replicas.max).ceil, @scale.min ].max
+        end
+
+        def sample(recorded, at)
+          Dash::Autoscale::Sample.new(value: recorded["value"], at: Dash::Autoscale::Timestamp.parse(recorded["at"]) || at, error: recorded["error"])
+        end
+
+        # The recorded readings against the current bounds. A guard with no recording at its
+        # index, or one of another source (hold_when reordered or changed), is left out.
+        def guards(recorded, at)
+          @scale.guards.each_with_index.filter_map do |guard, index|
+            next unless (reading = recorded_reading(recorded, guard, index))
+
+            Dash::Autoscale::Guard::Reading.new(index: index, source: guard.source, above: guard.above, sample: sample(reading, at))
+          end
+        end
+
+        def recorded_reading(recorded, guard, index)
+          Array(recorded).find { |reading| reading.is_a?(Hash) && reading["index"] == index && reading["source"] == guard.source }
+        end
+
+        # The current guards no replayed tick has a recording for.
+        def unrecorded_guards
+          return unless @scale.signal?
+
+          recorded = @ticks.filter_map { |_, decision| decision["inputs"] if decision["inputs"].is_a?(Hash) && decision["inputs"]["signal"].is_a?(Hash) }
+          return if recorded.empty?
+
+          missing = @scale.guards.each_with_index.count { |guard, index| recorded.none? { |inputs| recorded_reading(inputs["guards"], guard, index) } }
+          missing if missing.positive?
+        end
+
+        # { member host => unreachable since }, as recorded.
+        def unreachable(recorded)
+          return {} unless recorded.is_a?(Hash)
+
+          recorded.filter_map { |host, since| [ host, Dash::Autoscale::Timestamp.parse(since) ] if Dash::Autoscale::Timestamp.parse(since) }.to_h
+        end
+
+        def paused_until(recorded)
+          recorded == Dash::Autoscale::Pause::INDEFINITE ? :indefinite : Dash::Autoscale::Timestamp.parse(recorded)
+        end
+
+        def recorded_actions
+          ACTIONS.map { |action| @ticks.count { |_, decision| decision["action"] == action } }
+        end
+
+        # What ran after each recorded tick: where an action took it, else where it stood.
+        def recorded_counts
+          last = nil
+          @ticks.map { |at, decision| [ at, last = [ decision["to"], decision["from"] ].find { |count| count.is_a?(Integer) } || last ] }
+        end
+
+        # Container-seconds: each count held until the role's next tick.
+        def summary(scale_outs, scale_ins, counts)
+          counts = counts.reject { |_, count| count.nil? }
+          seconds = counts.each_cons(2).sum { |(at, count), (next_at, _)| count * (next_at - at) }
+
+          { scale_outs: scale_outs, scale_ins: scale_ins, peak: counts.map(&:last).max, minimum: counts.map(&:last).min, container_seconds: seconds.round }
+        end
+    end
+end

data/lib/dash/autoscale/state_store.rb

@@ -0,0 +1,132 @@
+require "json"
+
+# The controller's state, kept on `autoscale.controller` - the ops host that runs it - or,
+# without that key, on the primary role's first baseline host (never a member, which comes
+# and goes), so every operator's CLI and `dash mcp` read the same files: the heartbeat that
+# doubles as the single-controller lease, per-role state (unreachable-since times, last
+# scale-out and scale-in), the decision log, and pauses.
+#
+# Runs Dash::Commands::Autoscale through the SSHKit backend it is handed (call it from
+# inside `on(StateStore.host)`). A missing file is empty state; a malformed one is too, with
+# one warning per file, so a corrupted file never stops the controller. Reads give
+# string-keyed hashes, as JSON has them.
+class Dash::Autoscale::StateStore
+  def self.host(config = DASH.config)
+    config.autoscale.controller || config.primary_role.baseline_hosts.first
+  end
+
+  def initialize(backend, config: DASH.config)
+    @backend = backend
+    @config = config
+    @commands = Dash::Commands::Autoscale.new(config)
+    @warned = Set.new
+  end
+
+  def ensure_directory
+    execute @commands.ensure_directory
+  end
+
+  def heartbeat
+    read_object "heartbeat.json", @commands.read_heartbeat
+  end
+
+  # Written every tick, and during a long scale action, so it prints only with --verbose.
+  def write_heartbeat(heartbeat)
+    @backend.execute(*@commands.write_heartbeat(dump(heartbeat)), verbosity: :debug)
+  end
+
+  # The heartbeat, unless it names another controller: false then. A failed write raises.
+  def write_heartbeat_if_held(id, heartbeat)
+    @backend.capture(*@commands.write_heartbeat_if_held(id, dump(heartbeat)), verbosity: :debug).strip != "taken"
+  end
+
+  def state
+    read_object "state.json", @commands.read_state
+  end
+
+  # Whether `file` held something that is not a JSON object when last read.
+  def malformed?(file)
+    @warned.include?(file)
+  end
+
+  def write_state(state)
+    execute @commands.write_state(dump(state))
+  end
+
+  # The last `lines` entries of the decision log (of these roles', with `roles`), oldest first.
+  def decisions(lines:, roles: nil)
+    return [] if roles&.empty?
+
+    capture(@commands.read_decisions(lines: lines, roles: roles)).lines.filter_map { |line| parse("decisions.jsonl", line) if line.strip.present? }
+  end
+
+  # Decisions or hashes.
+  def append_decisions(decisions)
+    return if decisions.empty?
+
+    execute @commands.append_decisions(decisions.map { |decision| dump(decision.to_h) })
+  end
+
+  def decision_count
+    Integer(capture(@commands.count_decisions).strip.presence || "0", 10)
+  end
+
+  def trim_decisions(keep:)
+    execute @commands.trim_decisions(keep: keep)
+  end
+
+  # { role => pause }
+  def pauses
+    capture(@commands.read_pauses).lines.each_with_object({}) do |line, pauses|
+      path, json = line.chomp.split(":", 2)
+      # A `.json.tmp` an interrupted write left behind is no pause.
+      next if json.blank? || !path.end_with?(Dash::Commands::Autoscale::PAUSE_EXTENSION)
+
+      role = File.basename(path, Dash::Commands::Autoscale::PAUSE_EXTENSION)
+
+      pause = parse("pause/#{role}", json)
+      pauses[role] = pause if pause
+    end
+  end
+
+  def write_pause(role, pause)
+    execute @commands.write_pause(role, dump(pause))
+  end
+
+  def remove_pause(role)
+    execute @commands.remove_pause(role)
+  end
+
+  private
+    def read_object(file, command)
+      output = capture(command)
+      output.strip.empty? ? {} : parse(file, output) || {}
+    end
+
+    def parse(file, json)
+      value = JSON.parse(json)
+      value.is_a?(Hash) ? value : malformed(file)
+    rescue JSON::ParserError
+      malformed(file)
+    end
+
+    def malformed(file)
+      if @warned.add?(file)
+        warn "#{File.join(@config.app_directory, "autoscale", file)} on #{self.class.host(@config)} is not valid JSON, treating it as empty"
+      end
+
+      nil
+    end
+
+    def dump(value)
+      JSON.generate(Dash::Autoscale::Decision.json_safe(value))
+    end
+
+    def capture(command)
+      @backend.capture(*command, verbosity: :debug)
+    end
+
+    def execute(command)
+      @backend.execute(*command)
+    end
+end

data/lib/dash/autoscale/timestamp.rb

@@ -0,0 +1,15 @@
+require "time"
+
+# The times the controller's state files hold: ISO 8601 in UTC. A value that does not parse
+# reads as nil, because a hand-edited or half-written file must never stop the controller.
+module Dash::Autoscale::Timestamp
+  def self.parse(value)
+    Time.iso8601(value).utc if value.is_a?(String)
+  rescue ArgumentError
+    nil
+  end
+
+  def self.dump(time)
+    time&.utc&.iso8601
+  end
+end

data/lib/dash/autoscale/tunnel.rb

@@ -0,0 +1,48 @@
+require "net/ssh"
+
+# A local port forwarded over SSH through a `via` host to a database or Prometheus only the
+# app servers can reach. Its own Net::SSH session and event-loop thread - never one of
+# SSHKit's pooled connections, which nothing keeps looping - on 127.0.0.1 and an ephemeral
+# port net-ssh picks and returns.
+class Dash::Autoscale::Tunnel
+  ADDRESS = "127.0.0.1"
+  LOOP_WAIT = 0.1
+  CLOSE_WAIT = 2
+
+  attr_reader :port
+
+  def self.open(via, host, port, ssh: DASH.config.ssh)
+    new(via, host, port, user: ssh.user, ssh_options: ssh.options.except(:user))
+  end
+
+  def initialize(via, host, port, user:, ssh_options:, starter: ->(server, login, **options) { Net::SSH.start(server, login, options) })
+    @session = starter.call(via, user, **ssh_options)
+    begin
+      @port = @session.forward.local(ADDRESS, 0, host, port)
+    rescue StandardError
+      @session.close
+      raise
+    end
+    @closing = false
+    @thread = Thread.new { @session.loop(LOOP_WAIT) { !@closing } }
+    @thread.report_on_exception = false
+  end
+
+  def address
+    ADDRESS
+  end
+
+  # False once the session dropped: the client it carried has nothing to connect to.
+  def alive?
+    @thread.alive?
+  end
+
+  def close
+    @closing = true
+    @thread.join(CLOSE_WAIT)
+    @session.forward.cancel_local(port, ADDRESS)
+    @session.close
+  rescue IOError, Net::SSH::Exception
+    nil # the session is gone already
+  end
+end

data/lib/dash/cli/accessory.rb

@@ -130,6 +130,22 @@ class Dash::Cli::Accessory < Dash::Cli::Base
     end
   end
 
+  desc "stats [NAME]", "Show CPU, memory, network and block I/O of an accessory (a one-second sample; NAME=all for every accessory)"
+  option :json, type: :boolean, default: false, desc: "Print the stats per host as JSON"
+  def stats(name)
+    return error_on_missing_accessory(name) unless name == "all" || DASH.config.accessory(name)
+
+    accessories = name == "all" ? DASH.config.accessories : [ DASH.config.accessory(name) ]
+    snapshot = -> { Dash::Diagnostics::ContainerStats.new(hosts: [], accessories: accessories).to_h }
+    return puts_json(&snapshot) if options[:json]
+
+    pre_connect_if_required
+    entries = snapshot.call[:accessories]
+    return say "No host in scope runs #{name == "all" ? "an accessory" : name}", :yellow if entries.empty?
+
+    entries.each { |host| puts Dash::Cli::StatsTable.lines("Accessory #{host[:accessory]}", host) }
+  end
+
   desc "details [NAME]", "Show details about accessory on host (use NAME=all to show all accessories)"
   def details(name)
     quiet = options[:quiet]