solid_queue-flightdeck 0.6.0

data/app/models/flightdeck/metrics/line_chart.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  module Metrics
+    # Average completion time: area under a line, with gaps where no jobs
+    # finished at all.
+    class LineChart < Chart
+      DOT_RADIUS = 3
+
+      def coordinates
+        @coordinates ||= points.each_with_index.map do |point, index|
+          next nil if point.seconds.nil?
+
+          { x: round(plot_left + (slot_width * index) + (slot_width / 2.0)),
+            y: round(y_for(point.seconds)),
+            seconds: point.seconds,
+            title: point_title(point) }
+        end
+      end
+
+      # Contiguous runs of buckets that had data. Drawing them separately keeps
+      # a gap in the data looking like a gap, instead of a straight line across
+      # a period we know nothing about.
+      def segments
+        coordinates.chunk_while { |a, b| !a.nil? && !b.nil? }.filter_map do |chunk|
+          present = chunk.compact
+          present.size.positive? ? present : nil
+        end
+      end
+
+      def line_path(segment)
+        return "" if segment.empty?
+
+        segment.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
+      end
+
+      def area_path(segment)
+        return "" if segment.empty?
+
+        "#{line_path(segment)} L #{segment.last[:x]} #{plot_bottom} L #{segment.first[:x]} #{plot_bottom} Z"
+      end
+
+      def markers = coordinates.compact
+
+      def last_marker = markers.last
+
+      def any_data? = markers.any?
+
+      private
+        def raw_max = points.filter_map(&:seconds).max.to_f
+
+        # The zero gridline reads "0s" rather than "0ms": it is an axis origin,
+        # not a measurement.
+        def format_value(value)
+          value.zero? ? "0s" : Flightdeck::Duration.humanize(value)
+        end
+
+        def point_title(point)
+          "#{full_time(point.at)} · avg #{format_value(point.seconds)} to completion"
+        end
+    end
+  end
+end

data/app/models/flightdeck/metrics/series.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  module Metrics
+    # Time series for the Overview charts, computed live from the Solid Queue
+    # tables — no rollup tables, no background aggregation.
+    #
+    # Every series is gap-filled in Ruby so a chart always has a complete axis:
+    # a bucket with no rows is a real zero (or a real "no data"), not a missing
+    # point that would make the chart lie about its own shape.
+    class Series
+      Window = Struct.new(:key, :label, :duration, :bucket_seconds, :subtitle, :tick_every,
+                          :tick_format, keyword_init: true) do
+        def bucket_count = (duration / bucket_seconds).to_i
+      end
+
+      WINDOWS = {
+        "1h" => Window.new(key: "1h", label: "1H", duration: 1.hour, bucket_seconds: 5.minutes.to_i,
+                           subtitle: "per 5 minutes", tick_every: 3, tick_format: "%H:%M"),
+        "24h" => Window.new(key: "24h", label: "24H", duration: 24.hours, bucket_seconds: 1.hour.to_i,
+                            subtitle: "per hour", tick_every: 4, tick_format: "%H:%M"),
+        "7d" => Window.new(key: "7d", label: "7D", duration: 7.days, bucket_seconds: 6.hours.to_i,
+                           subtitle: "per 6 hours", tick_every: 4, tick_format: "%-m/%-d")
+      }.freeze
+
+      DEFAULT_WINDOW = "24h"
+      SPARKLINE_HOURS = 16
+      SPARKLINE_BUCKET = 1.hour.to_i
+
+      Point = Struct.new(:at, :succeeded, :failed, :seconds, keyword_init: true) do
+        def total = succeeded.to_i + failed.to_i
+        def blank? = seconds.nil?
+      end
+
+      attr_reader :window, :now
+
+      def self.window_for(key)
+        WINDOWS.fetch(key.to_s, WINDOWS.fetch(DEFAULT_WINDOW))
+      end
+
+      def initialize(window: DEFAULT_WINDOW, now: Time.current)
+        @window = window.is_a?(Window) ? window : self.class.window_for(window)
+        @now = now.utc
+      end
+
+      # Bucket start times, aligned to absolute epoch multiples so that the same
+      # bucket has the same boundaries in every process and every request.
+      def buckets
+        @buckets ||= begin
+          last = align(@now.to_i)
+          count = window.bucket_count
+          Array.new(count) { |i| last - ((count - 1 - i) * window.bucket_seconds) }
+        end
+      end
+
+      def starts_at = Time.at(buckets.first).utc
+      def ends_at = Time.at(buckets.last + window.bucket_seconds).utc
+
+      # Jobs finished (succeeded) and executions failed, per bucket.
+      #
+      # A job that failed has no finished_at — Solid Queue leaves it null and
+      # records a failed_executions row — so the two series never double count.
+      def throughput
+        cached(:throughput) do
+          succeeded = grouped_count(SolidQueue::Job.where(finished_at: starts_at..), :finished_at)
+          failed = grouped_count(SolidQueue::FailedExecution.where(created_at: starts_at..), :created_at,
+                                 table: SolidQueue::FailedExecution.table_name)
+
+          buckets.map do |bucket|
+            Point.new(at: Time.at(bucket).utc,
+                      succeeded: succeeded.fetch(bucket, 0),
+                      failed: failed.fetch(bucket, 0))
+          end
+        end
+      end
+
+      # Average wall-clock time from enqueue to finish, per bucket.
+      #
+      # This is time *to completion*, not time to start: Solid Queue deletes the
+      # claimed_executions row when a job finishes, so the moment a job actually
+      # started is not recoverable after the fact and no honest historical
+      # time-to-start series can be built from these tables.
+      def completion_time
+        cached(:completion_time) do
+          averages = SolidQueue::Job
+            .where(finished_at: starts_at..)
+            .group(TimeBucket.sql("#{jobs_table}.finished_at", window.bucket_seconds))
+            .average(TimeBucket.elapsed_sql("#{jobs_table}.created_at", "#{jobs_table}.finished_at"))
+            .transform_keys(&:to_i)
+
+          buckets.map do |bucket|
+            value = averages[bucket]
+            Point.new(at: Time.at(bucket).utc, seconds: value&.to_f)
+          end
+        end
+      end
+
+      def total_succeeded = throughput.sum(&:succeeded)
+      def total_failed = throughput.sum(&:failed)
+
+      def empty? = throughput.all? { |point| point.total.zero? }
+
+      # Finished-job retention shorter than the window means the oldest buckets
+      # are missing rows that really did happen. The chart says so rather than
+      # showing a decline that is an artefact of purging.
+      def truncated_by_retention?
+        retention.present? && retention < window.duration
+      end
+
+      def retention
+        return nil unless SolidQueue.respond_to?(:clear_finished_jobs_after)
+        return nil unless SolidQueue.preserve_finished_jobs?
+
+        SolidQueue.clear_finished_jobs_after
+      end
+
+      # One query for every queue's hourly finished count, fanned out in Ruby.
+      # Used by the queue cards and the Overview mini-table.
+      def self.queue_sparklines(now: Time.current, hours: SPARKLINE_HOURS)
+        Flightdeck::Cache.fetch("sparklines", hours, align_to(now.to_i, SPARKLINE_BUCKET),
+                                expires_in: Flightdeck.config.chart_cache_ttl) do
+          last = align_to(now.to_i, SPARKLINE_BUCKET)
+          slots = Array.new(hours) { |i| last - ((hours - 1 - i) * SPARKLINE_BUCKET) }
+
+          counts = SolidQueue::Job
+            .where(finished_at: Time.at(slots.first).utc..)
+            .group(:queue_name)
+            .group(TimeBucket.sql("#{SolidQueue::Job.table_name}.finished_at", SPARKLINE_BUCKET))
+            .count
+
+          fanned = Hash.new { |hash, key| hash[key] = Array.new(hours, 0) }
+          counts.each do |(queue_name, bucket), count|
+            index = slots.index(bucket.to_i)
+            next unless index
+
+            fanned[queue_name][index] = count
+          end
+          fanned.default = nil
+          fanned
+        end
+      end
+
+      def self.align_to(epoch, seconds) = (epoch / seconds) * seconds
+
+      private
+        def jobs_table = SolidQueue::Job.table_name
+
+        def align(epoch) = self.class.align_to(epoch, window.bucket_seconds)
+
+        def grouped_count(relation, column, table: SolidQueue::Job.table_name)
+          relation
+            .group(TimeBucket.sql("#{table}.#{column}", window.bucket_seconds))
+            .count
+            .transform_keys(&:to_i)
+        end
+
+        # Keyed on the aligned bucket so the cache turns over with the buckets
+        # themselves rather than at an arbitrary moment.
+        def cached(name, &block)
+          Flightdeck::Cache.fetch("series", name, window.key, buckets.last,
+                                  expires_in: Flightdeck.config.chart_cache_ttl, &block)
+        end
+    end
+  end
+end

data/app/models/flightdeck/metrics/sparkline.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  module Metrics
+    # Tiny area+line for the queue cards and the Overview mini-table.
+    class Sparkline
+      WIDTH = 100
+      HEIGHT = 30
+
+      attr_reader :values
+
+      def initialize(values)
+        @values = Array(values).map(&:to_i)
+      end
+
+      def width = WIDTH
+      def height = HEIGHT
+      def any? = values.any? && values.max.positive?
+
+      def coordinates
+        @coordinates ||= begin
+          max = [ values.max.to_i, 1 ].max
+          step = values.size > 1 ? WIDTH.to_f / (values.size - 1) : WIDTH.to_f
+
+          values.each_with_index.map do |value, index|
+            { x: (index * step).round(2),
+              y: (HEIGHT - 2 - ((value.to_f / max) * (HEIGHT - 4))).round(2) }
+          end
+        end
+      end
+
+      def line_path
+        coordinates.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
+      end
+
+      def area_path
+        return "" if coordinates.empty?
+
+        "#{line_path} L #{coordinates.last[:x]} #{HEIGHT} L #{coordinates.first[:x]} #{HEIGHT} Z"
+      end
+
+      def last_point = coordinates.last
+    end
+  end
+end

data/app/models/flightdeck/metrics/time_bucket.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  module Metrics
+    # The only adapter-specific SQL in Flightdeck.
+    #
+    # Charts bucket timestamps into fixed-width windows. Doing that portably
+    # means converting a timestamp to a UTC epoch integer and dividing — never
+    # date_trunc, which does not exist everywhere and drags the server's
+    # timezone into the answer. Everything downstream is an integer number of
+    # seconds since the epoch, in UTC; the display timezone is applied at render
+    # time and nowhere else.
+    module TimeBucket
+      class UnsupportedAdapter < StandardError; end
+
+      # Identifiers are built by Flightdeck, never by a request, but they are
+      # still validated before being interpolated so that can never change by
+      # accident.
+      IDENTIFIER = /\A[a-z_][a-z0-9_]*(\.[a-z_][a-z0-9_]*)?\z/
+
+      class << self
+        def adapter_name
+          SolidQueue::Record.connection.adapter_name.to_s
+        end
+
+        # Normalises the adapter into the three SQL dialects we support.
+        def family(name = adapter_name)
+          case name.to_s.downcase
+          when /postgres/ then :postgresql
+          when /mysql|trilogy/ then :mysql
+          when /sqlite/ then :sqlite
+          else
+            raise UnsupportedAdapter,
+                  "Flightdeck cannot bucket time on the #{name} adapter. " \
+                  "Supported adapters: PostgreSQL, MySQL/Trilogy, SQLite."
+          end
+        end
+
+        # Seconds since the UTC epoch, as a number the database can do
+        # arithmetic on.
+        def epoch(column, family: self.family)
+          col = identifier!(column)
+
+          case family
+          when :postgresql then "EXTRACT(EPOCH FROM #{col})"
+          when :mysql then "UNIX_TIMESTAMP(#{col})"
+          when :sqlite then "CAST(strftime('%s', #{col}) AS INTEGER)"
+          else raise UnsupportedAdapter, "unknown family #{family.inspect}"
+          end
+        end
+
+        # The epoch second at which `column`'s bucket starts.
+        #
+        # SQLite gets integer division rather than FLOOR() on purpose: floor() is
+        # only available when SQLite is compiled with math functions, and
+        # integer division is exact for the post-1970 timestamps we deal with.
+        def bucket(column, seconds, family: self.family)
+          seconds = interval!(seconds)
+          col = identifier!(column)
+
+          case family
+          when :postgresql then "FLOOR(EXTRACT(EPOCH FROM #{col}) / #{seconds}) * #{seconds}"
+          when :mysql then "FLOOR(UNIX_TIMESTAMP(#{col}) / #{seconds}) * #{seconds}"
+          when :sqlite then "(CAST(strftime('%s', #{col}) AS INTEGER) / #{seconds}) * #{seconds}"
+          else raise UnsupportedAdapter, "unknown family #{family.inspect}"
+          end
+        end
+
+        def sql(column, seconds) = Arel.sql(bucket(column, seconds))
+
+        # Seconds between two timestamp columns, for averaging in SQL.
+        def elapsed(from_column, to_column, family: self.family)
+          "(#{epoch(to_column, family: family)} - #{epoch(from_column, family: family)})"
+        end
+
+        def elapsed_sql(from_column, to_column) = Arel.sql(elapsed(from_column, to_column))
+
+        private
+          def identifier!(column)
+            value = column.to_s
+            return value if IDENTIFIER.match?(value)
+
+            raise ArgumentError, "#{column.inspect} is not a plain column identifier"
+          end
+
+          def interval!(seconds)
+            value = Integer(seconds)
+            return value if value.positive?
+
+            raise ArgumentError, "bucket width must be a positive number of seconds"
+          end
+      end
+    end
+  end
+end

data/app/models/flightdeck/overview.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  # Everything the Overview page shows, assembled from the same query objects
+  # the rest of the dashboard uses.
+  class Overview
+    WINDOW = 24.hours
+    RECENT_FAILURES = 5
+    TOP_QUEUES = 6
+
+    # `value` is always display-ready text — a delimited count, a duration, or
+    # "—" — so the view prints it without asking what kind of number it was.
+    # trend is :up / :down / :flat and says which way the number moved, while
+    # `good` says whether that movement is a good thing — more throughput is
+    # good, more failures are not.
+    Tile = Struct.new(:label, :value, :unit, :detail, :trend, :good, keyword_init: true)
+
+    attr_reader :now
+
+    def initialize(now: Time.current)
+      @now = now
+    end
+
+    def tiles
+      [ processed_tile, failed_tile, ready_tile, scheduled_tile, in_progress_tile, oldest_ready_tile ]
+    end
+
+    def series(window: Metrics::Series::DEFAULT_WINDOW)
+      @series ||= {}
+      @series[window.to_s] ||= Metrics::Series.new(window: window, now: now)
+    end
+
+    def queues
+      @queues ||= QueueStats.new.rows.sort_by { |row| -row.depth }.first(TOP_QUEUES)
+    end
+
+    def max_queue_depth
+      @max_queue_depth ||= [ queues.map(&:depth).max.to_i, 1 ].max
+    end
+
+    def sparklines
+      @sparklines ||= Metrics::Series.queue_sparklines(now: now)
+    end
+
+    def sparkline_for(queue_name)
+      Metrics::Sparkline.new(sparklines[queue_name] || [])
+    end
+
+    def recent_failures
+      @recent_failures ||= JobsQuery.new(state: :failed, limit: RECENT_FAILURES).rows
+    end
+
+    def failed_count
+      @failed_count ||= JobsQuery.new(state: :failed).count
+    end
+
+    def registry
+      @registry ||= ProcessRegistry.new
+    end
+
+    # --- counts ---------------------------------------------------------------
+
+    def processed_24h
+      @processed_24h ||= counted("processed", WINDOW) { SolidQueue::Job.where(finished_at: (now - WINDOW)..now) }
+    end
+
+    def processed_prior_24h
+      @processed_prior_24h ||= counted("processed_prior", WINDOW) do
+        SolidQueue::Job.where(finished_at: (now - (WINDOW * 2))...(now - WINDOW))
+      end
+    end
+
+    def failed_24h
+      @failed_24h ||= counted("failed", WINDOW) { SolidQueue::FailedExecution.where(created_at: (now - WINDOW)..now) }
+    end
+
+    def ready_now = @ready_now ||= counted("ready") { SolidQueue::ReadyExecution.all }
+    def scheduled = @scheduled ||= counted("scheduled") { SolidQueue::ScheduledExecution.all }
+    def in_progress = @in_progress ||= counted("in_progress") { SolidQueue::ClaimedExecution.all }
+
+    def next_scheduled_at
+      @next_scheduled_at ||= Cache.fetch("overview", "next_scheduled", expires_in: count_ttl) do
+        SolidQueue::ScheduledExecution.minimum(:scheduled_at)
+      end
+    end
+
+    def oldest_ready_at
+      @oldest_ready_at ||= Cache.fetch("overview", "oldest_ready", expires_in: count_ttl) do
+        SolidQueue::ReadyExecution.minimum(:created_at)
+      end
+    end
+
+    def oldest_ready_age
+      oldest_ready_at && now - oldest_ready_at
+    end
+
+    # Total worker capacity, summed from each worker's registered thread pool
+    # size. Returns nil when no worker reported one, so the tile can omit the
+    # denominator rather than divide by a number it invented.
+    #
+    # A worker that has stopped sending heartbeats is not capacity, whatever its
+    # metadata still says, so dead workers are excluded — otherwise utilization
+    # reads low precisely when the fleet is in trouble.
+    def slots
+      @slots ||= begin
+        sizes = registry.nodes.filter_map do |node|
+          next unless node.claims_executions? && !node.dead?
+
+          Integer(node.metadata.symbolize_keys[:thread_pool_size], exception: false)
+        end
+        sizes.any? ? sizes.sum : nil
+      end
+    end
+
+    def utilization
+      return nil if slots.nil? || slots.zero?
+
+      (in_progress.to_f / slots * 100).round
+    end
+
+    def failure_rate
+      total = processed_24h + failed_24h
+      return nil if total.zero?
+
+      (failed_24h.to_f / total * 100).round(1)
+    end
+
+    def processed_delta
+      return nil if processed_prior_24h.zero?
+
+      ((processed_24h - processed_prior_24h).to_f / processed_prior_24h * 100).round(1)
+    end
+
+    private
+      def count_ttl = Flightdeck.config.poll_interval
+
+      def counted(name, window = nil, &relation)
+        Cache.fetch("overview", name, window&.to_i, expires_in: count_ttl) do
+          relation.call.limit(Flightdeck.config.count_cap).count
+        end
+      end
+
+      def processed_tile
+        delta = processed_delta
+
+        Tile.new(
+          label: "Processed · 24h",
+          value: number(processed_24h),
+          detail: delta.nil? ? "no prior window to compare" : "#{delta.abs}% vs prior 24h",
+          trend: trend_for(delta),
+          good: delta.nil? || delta >= 0
+        )
+      end
+
+      def failed_tile
+        rate = failure_rate
+
+        Tile.new(
+          label: "Failed · 24h",
+          value: number(failed_24h),
+          detail: rate.nil? ? "nothing finished yet" : "#{rate}% failure rate",
+          trend: failed_24h.positive? ? :up : :flat,
+          good: failed_24h.zero?
+        )
+      end
+
+      def ready_tile
+        Tile.new(label: "Ready now", value: number(ready_now), detail: "waiting to be claimed", trend: :flat, good: true)
+      end
+
+      def scheduled_tile
+        detail =
+          if next_scheduled_at.nil? then "nothing scheduled"
+          elsif next_scheduled_at <= now then "due now"
+          else "next due in #{Duration.humanize(next_scheduled_at - now)}"
+          end
+
+        Tile.new(label: "Scheduled", value: number(scheduled), detail: detail, trend: :flat, good: true)
+      end
+
+      def in_progress_tile
+        Tile.new(
+          label: "In progress",
+          value: number(in_progress),
+          unit: slots ? "/ #{number(slots)} slots" : nil,
+          detail: utilization ? "#{utilization}% utilization" : "no worker capacity reported",
+          trend: :flat,
+          good: true
+        )
+      end
+
+      def oldest_ready_tile
+        Tile.new(
+          label: "Oldest ready",
+          value: oldest_ready_age ? Duration.humanize(oldest_ready_age) : "—",
+          detail: oldest_ready_age ? "longest a job has waited" : "nothing waiting",
+          trend: :flat,
+          good: true
+        )
+      end
+
+      def number(value) = ActiveSupport::NumberHelper.number_to_delimited(value)
+
+      def trend_for(delta)
+        return :flat if delta.nil? || delta.abs < 0.05
+
+        delta.positive? ? :up : :down
+      end
+  end
+end

data/app/models/flightdeck/process_registry.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  # The Solid Queue fleet: every registered process, grouped supervisor →
+  # children, with heartbeat freshness and claimed counts.
+  #
+  # Two queries total, regardless of fleet size.
+  class ProcessRegistry
+    # Freshness is measured against Solid Queue's own liveness threshold rather
+    # than a number of our own, so what Flightdeck calls dead is exactly what
+    # the pruner will collect.
+    FRESH_FRACTION = 0.2
+
+    Node = Struct.new(:process, :claimed_count, :children, keyword_init: true) do
+      def id = process.id
+      def kind = process.kind
+      def name = process.name
+      def hostname = process.hostname
+      def pid = process.pid
+      def last_heartbeat_at = process.last_heartbeat_at
+      def supervisor? = kind == "Supervisor"
+      def claims_executions? = kind == "Worker"
+
+      def heartbeat_age
+        Time.current - last_heartbeat_at
+      end
+
+      def freshness
+        threshold = SolidQueue.process_alive_threshold
+        age = heartbeat_age
+
+        if age >= threshold then :dead
+        elsif age >= threshold * FRESH_FRACTION then :stale
+        else :fresh
+        end
+      end
+
+      def dead? = freshness == :dead
+      def prunable? = dead?
+
+      def metadata
+        process.metadata.presence || {}
+      end
+
+      # Human summary of what this process was configured to do. Keys differ per
+      # process kind, so known ones are formatted and anything else is shown
+      # verbatim rather than dropped.
+      def config_summary
+        parts = []
+        data = metadata.symbolize_keys
+
+        parts << "queues: #{data[:queues]}" if data[:queues].present?
+        parts << "#{data[:thread_pool_size]} threads" if data[:thread_pool_size].present?
+        parts << "batch #{data[:batch_size]}" if data[:batch_size].present?
+        parts << "every #{data[:polling_interval]}s" if data[:polling_interval].present?
+        parts << "#{Array(data[:recurring_schedule]).size} recurring tasks" if data[:recurring_schedule].present?
+
+        known = %i[queues thread_pool_size batch_size polling_interval recurring_schedule
+                   concurrency_maintenance_interval]
+        data.except(*known).each { |key, value| parts << "#{key}: #{value}" }
+
+        parts.join(" · ")
+      end
+    end
+
+    def nodes
+      @nodes ||= build_nodes
+    end
+
+    # Supervisors with their children nested; anything unsupervised (or whose
+    # supervisor row has gone) is listed flat so it can never be hidden.
+    def tree
+      @tree ||= begin
+        by_supervisor = nodes.group_by { |node| node.process.supervisor_id }
+        known_ids = nodes.map(&:id).to_set
+
+        roots = nodes.select { |node| node.process.supervisor_id.nil? || !known_ids.include?(node.process.supervisor_id) }
+        roots.each { |root| root.children = by_supervisor.fetch(root.id, []) }
+        roots
+      end
+    end
+
+    def dead = nodes.select(&:dead?)
+    def any_dead? = dead.any?
+    def dead_claimed_count = dead.sum(&:claimed_count)
+    def total = nodes.size
+
+    def find(id)
+      nodes.find { |node| node.id == id.to_i }
+    end
+
+    private
+      def build_nodes
+        processes = SolidQueue::Process.order(:kind, :id).to_a
+        claimed = SolidQueue::ClaimedExecution.group(:process_id).count
+
+        processes.map do |process|
+          Node.new(process: process, claimed_count: claimed.fetch(process.id, 0), children: [])
+        end
+      end
+  end
+end