flightdeck 0.5.0

data/app/models/flightdeck/job_detail.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  # Everything the job show page needs, assembled from Solid Queue's own models.
+  #
+  # Arguments are shown exactly as they were stored. They are never handed to
+  # ActiveJob for deserialization: that would load host application classes (and
+  # run their code) just to render a dashboard.
+  class JobDetail
+    ARGUMENTS_DISPLAY_LIMIT = 10.kilobytes
+
+    # A frame belongs to the application unless it lives in an installed gem or
+    # in Ruby's own library directories.
+    VENDOR_FRAME = %r{
+      (?:\A|/)(?:gems|ruby|rubygems|bundler|vendor/bundle)/ |
+      \A<internal: | \A/usr/lib/ruby/
+    }x
+
+    attr_reader :job, :state, :execution, :process
+
+    def self.find(id)
+      job = SolidQueue::Job.find(id)
+      annotation = JobRow.annotate([ job.id ])[job.id]
+
+      new(job: job, annotation: annotation)
+    end
+
+    def initialize(job:, annotation: nil)
+      @job = job
+      @state = resolve_state(annotation)
+      @execution = load_execution(annotation)
+      @process = load_process
+    end
+
+    def id = job.id
+    def class_name = job.class_name
+    def queue_name = job.queue_name
+    def priority = job.priority
+    def active_job_id = job.active_job_id
+    def concurrency_key = job.concurrency_key
+    def enqueued_at = job.created_at
+    def scheduled_at = job.scheduled_at
+    def finished_at = job.finished_at
+
+    def failed? = state == :failed
+    def retryable? = failed?
+    def discardable? = state != :in_progress && state != :finished
+
+    def failed_execution
+      execution if failed?
+    end
+
+    # --- arguments ----------------------------------------------------------
+
+    # `job.arguments` is already a Ruby structure (Solid Queue stores it with a
+    # JSON coder); pretty-printing it is a formatting step, not deserialization.
+    def arguments_json
+      @arguments_json ||= begin
+        JSON.pretty_generate(job.arguments)
+      rescue StandardError
+        job.arguments.to_s
+      end
+    end
+
+    def arguments_truncated?
+      arguments_json.bytesize > ARGUMENTS_DISPLAY_LIMIT
+    end
+
+    def arguments_preview
+      return arguments_json unless arguments_truncated?
+
+      arguments_json.byteslice(0, ARGUMENTS_DISPLAY_LIMIT).scrub("")
+    end
+
+    def arguments_bytes = arguments_json.bytesize
+
+    # --- error --------------------------------------------------------------
+
+    def error_class
+      failed_execution&.exception_class
+    end
+
+    def error_message
+      failed_execution&.message
+    end
+
+    def backtrace
+      @backtrace ||= Array(failed_execution&.backtrace).first(Flightdeck.config.backtrace_lines)
+    end
+
+    def backtrace_total = Array(failed_execution&.backtrace).size
+
+    def backtrace_truncated? = backtrace_total > backtrace.size
+
+    def backtrace_frames
+      backtrace.map { |line| { line: line, app: app_frame?(line) } }
+    end
+
+    def app_frame?(line)
+      !VENDOR_FRAME.match?(line.to_s)
+    end
+
+    # --- timeline -----------------------------------------------------------
+
+    Event = Struct.new(:label, :at, :detail, :status, keyword_init: true)
+
+    # Built only from timestamps Solid Queue actually keeps. Steps whose row has
+    # already been consumed (a finished job's claim, say) are simply absent
+    # rather than invented.
+    def timeline
+      events = [ Event.new(label: "Enqueued", at: enqueued_at, detail: "queue #{queue_name}", status: :ok) ]
+
+      if scheduled_at.present? && scheduled_at > enqueued_at
+        events << Event.new(label: "Scheduled", at: scheduled_at,
+                            detail: scheduled_at.future? ? "not yet due" : "became due", status: :ok)
+      end
+
+      case state
+      when :ready
+        events << Event.new(label: "Ready", at: execution&.created_at, detail: "waiting for a worker", status: :now)
+      when :blocked
+        events << Event.new(label: "Blocked", at: execution&.created_at,
+                            detail: "concurrency key #{job.concurrency_key}", status: :now)
+      when :in_progress
+        events << Event.new(label: "Claimed", at: execution&.created_at, detail: process_label, status: :now)
+      when :failed
+        events << Event.new(label: "Failed", at: execution&.created_at, detail: error_class, status: :bad)
+        events << Event.new(label: "Awaiting decision", at: nil,
+                            detail: "retry or discard", status: :now)
+      end
+
+      events << Event.new(label: "Finished", at: finished_at, detail: nil, status: :ok) if finished_at.present?
+      events.select { |event| event.at.present? || event.status == :now }
+    end
+
+    def process_label
+      return nil unless process
+
+      [ process.name, ("pid #{process.pid}" if process.pid) ].compact.join(" · ")
+    end
+
+    private
+      def resolve_state(annotation)
+        return :finished if job.finished_at.present?
+        return annotation[:state] if annotation
+
+        :unknown
+      end
+
+      def load_execution(annotation)
+        return nil unless annotation
+
+        model = JobRow.model_for(state)
+        model&.find_by(id: annotation[:execution_id])
+      end
+
+      def load_process
+        return nil unless execution.respond_to?(:process_id) && execution.process_id
+
+        SolidQueue::Process.find_by(id: execution.process_id)
+      end
+  end
+end

data/app/models/flightdeck/job_row.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  # One row of a jobs list. Deliberately dumb: JobsQuery assembles these from
+  # bounded queries and the view asks them formatting questions.
+  class JobRow
+    # Every table a job can be sitting in, in the order we report them. A job is
+    # only ever in one of these at a time, but we probe all five so a row is
+    # never silently mislabelled during a transition.
+    EXECUTION_MODELS = {
+      ready: "SolidQueue::ReadyExecution",
+      scheduled: "SolidQueue::ScheduledExecution",
+      in_progress: "SolidQueue::ClaimedExecution",
+      blocked: "SolidQueue::BlockedExecution",
+      failed: "SolidQueue::FailedExecution"
+    }.freeze
+
+    class << self
+      def model_for(state)
+        name = EXECUTION_MODELS[state.to_sym]
+        name && name.constantize
+      end
+
+      # Resolves the state of an already-loaded page of jobs with one
+      # `WHERE job_id IN (page_ids)` lookup per executions table. Returns
+      # { job_id => { state:, execution_id:, process_id: } }.
+      def annotate(job_ids)
+        job_ids = Array(job_ids).compact.uniq
+        return {} if job_ids.empty?
+
+        EXECUTION_MODELS.each_with_object({}) do |(state, model_name), found|
+          model = model_name.constantize
+          columns = [ :id, :job_id ]
+          columns << :process_id if model.column_names.include?("process_id")
+
+          model.where(job_id: job_ids).pluck(*columns).each do |values|
+            job_id = values[1]
+            next if found.key?(job_id)
+
+            found[job_id] = { state: state, execution_id: values[0], process_id: values[2] }
+          end
+        end
+      end
+    end
+
+    attr_reader :job, :state, :execution, :args_preview, :error_summary, :process
+
+    def initialize(job:, state:, execution: nil, args_preview: nil, error_summary: nil, process: nil)
+      @job = job
+      @state = state
+      @execution = execution
+      @args_preview = args_preview
+      @error_summary = error_summary || ErrorSummary.none
+      @process = process
+    end
+
+    def id = job.id
+    def class_name = job.class_name
+    def queue_name = job.queue_name
+    def priority = job.priority
+    def active_job_id = job.active_job_id
+    def concurrency_key = job.concurrency_key
+    def enqueued_at = job.created_at
+    def scheduled_at = job.scheduled_at
+    def finished_at = job.finished_at
+
+    def failed? = state == :failed
+    def discardable? = state != :in_progress
+
+    # The moment the job first became eligible to run — what "waiting" is
+    # measured from, so a job scheduled for next week is not reported as having
+    # waited a week.
+    def due_at
+      scheduled_at || enqueued_at
+    end
+
+    def execution_started_at
+      execution&.created_at if state == :in_progress
+    end
+
+    def failed_at
+      execution&.created_at if failed?
+    end
+
+    # Attempt number, when ActiveJob recorded one in the serialized payload. We
+    # only ever see a truncated prefix of the arguments, so this is best-effort
+    # by design and returns nil rather than guessing.
+    def attempts
+      value = args_preview.to_s[/"executions"\s*:\s*(\d+)/, 1]
+      value && value.to_i + 1
+    end
+  end
+end

data/app/models/flightdeck/jobs_query.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  # Builds every jobs list in Flightdeck.
+  #
+  # Rule zero: everything goes through SolidQueue's own models, so multi-database
+  # routing comes for free and the host's connection is never touched.
+  #
+  # Two shapes of query, chosen by state:
+  #   * an execution-backed state drives from its own table (that is what makes
+  #     the state true) and then loads the matching jobs;
+  #   * "all" and "finished" drive from solid_queue_jobs and have their state
+  #     annotated afterwards.
+  #
+  # The `arguments` and `error` columns are never in a list's driving query.
+  # Both are fetched afterwards for the current page only, and truncated in SQL
+  # so a page of pathological payloads still costs a bounded number of bytes.
+  class JobsQuery
+    STATES = %i[all ready scheduled in_progress blocked finished failed].freeze
+
+    # Enough that the ActiveJob envelope (job_class, job_id, queue_name…) is
+    # comfortably past and the arguments themselves are in the prefix.
+    ARGUMENTS_PREVIEW_BYTES = 800
+    ERROR_PREVIEW_BYTES = 1_000
+
+    class InvalidState < ArgumentError; end
+
+    attr_reader :state, :class_name, :queue_name, :q, :before_id, :limit, :count_cap
+
+    def initialize(state: :all, class_name: nil, queue_name: nil, q: nil, before_id: nil,
+                   limit: nil, count_cap: nil)
+      @state = normalize_state(state)
+      @class_name = class_name.presence
+      @queue_name = queue_name.presence
+      @q = q.presence
+      @before_id = before_id.presence && Integer(before_id, exception: false)
+      @limit = (limit || Flightdeck.config.per_page).to_i.clamp(1, 200)
+      @count_cap = (count_cap || Flightdeck.config.count_cap).to_i
+    end
+
+    def rows
+      @rows ||= build_rows
+    end
+
+    def next_cursor
+      rows
+      @next_cursor
+    end
+
+    def next_page? = next_cursor.present?
+
+    # Capped so that a jobs table with tens of millions of rows cannot turn a
+    # page render into a full scan. Views render `count_capped?` as "500,000+".
+    #
+    # Cached for one poll interval: the state tabs and the sidebar issue several
+    # of these per render, and a count that is at most one poll stale is exactly
+    # as fresh as the page it appears on. Mutations wrap themselves in
+    # Cache.bypass so an action's own response never shows pre-action numbers.
+    def count
+      @count ||= Flightdeck::Cache.fetch("count", state, count_cap, cache_filters,
+                                         expires_in: Flightdeck.config.poll_interval) do
+        filtered_relation.limit(count_cap).count
+      end
+    end
+
+    def count_capped? = count >= count_cap
+
+    def filters?
+      class_name.present? || queue_name.present? || q.present?
+    end
+
+    # The filtered relation with no ordering, limit or cursor — what bulk
+    # "apply to all matching" re-runs server-side.
+    def filtered_relation
+      @filtered_relation ||= apply_filters(model.all)
+    end
+
+    def model
+      @model ||= JobRow.model_for(state) || SolidQueue::Job
+    end
+
+    def execution_backed? = model != SolidQueue::Job
+
+    # Counts for the state tabs, each capped independently.
+    def self.state_counts(**filters)
+      STATES.index_with do |state|
+        query = new(**filters.except(:state, :before_id), state: state)
+        { count: query.count, capped: query.count_capped? }
+      end
+    end
+
+    private
+      JOB_LIST_COLUMNS = -> { SolidQueue::Job.column_names - [ "arguments" ] }
+
+      # Only the filters change what a count means — the cursor and page size do
+      # not, so they are deliberately absent from the key.
+      def cache_filters
+        { class_name: class_name, queue_name: queue_name, q: q }
+      end
+
+      def normalize_state(value)
+        state = value.presence&.to_sym || :all
+        raise InvalidState, "unknown job state #{value.inspect}" unless STATES.include?(state)
+
+        state
+      end
+
+      def build_rows
+        page = paginated_records
+        @next_cursor = page.size > limit ? page[limit - 1].id : nil
+        page = page.first(limit)
+
+        return [] if page.empty?
+
+        execution_backed? ? rows_from_executions(page) : rows_from_jobs(page)
+      end
+
+      def paginated_records
+        relation = filtered_relation
+          .reorder(model.arel_table[:id].desc)
+          .limit(limit + 1)
+
+        relation = relation.where(model.arel_table[:id].lt(before_id)) if before_id
+        relation.select(driving_columns).to_a
+      end
+
+      # Never `SELECT *`: it would drag `arguments` (jobs) or `error`
+      # (failed_executions) through every list query.
+      def driving_columns
+        names = model.column_names - [ "arguments", "error" ]
+        names.map { |name| model.arel_table[name] }
+      end
+
+      def rows_from_executions(executions)
+        job_ids = executions.map(&:job_id)
+        jobs = jobs_by_id(job_ids)
+        previews = argument_previews(job_ids)
+        errors = state == :failed ? error_previews(executions.map(&:id)) : {}
+        processes = state == :in_progress ? processes_by_id(executions.map(&:process_id)) : {}
+
+        executions.filter_map do |execution|
+          job = jobs[execution.job_id]
+          next unless job
+
+          JobRow.new(
+            job: job,
+            state: state,
+            execution: execution,
+            args_preview: previews[job.id],
+            error_summary: errors[execution.id],
+            process: processes[execution.try(:process_id)]
+          )
+        end
+      end
+
+      def rows_from_jobs(jobs)
+        job_ids = jobs.map(&:id)
+        previews = argument_previews(job_ids)
+        annotations = state == :finished ? {} : JobRow.annotate(job_ids)
+
+        jobs.map do |job|
+          annotation = annotations[job.id]
+
+          JobRow.new(
+            job: job,
+            state: state_for(job, annotation),
+            args_preview: previews[job.id]
+          )
+        end
+      end
+
+      def state_for(job, annotation)
+        return :finished if job.finished_at.present?
+        return annotation[:state] if annotation
+
+        :unknown
+      end
+
+      def jobs_by_id(job_ids)
+        SolidQueue::Job.where(id: job_ids).select(JOB_LIST_COLUMNS.call).index_by(&:id)
+      end
+
+      def processes_by_id(process_ids)
+        ids = process_ids.compact.uniq
+        return {} if ids.empty?
+
+        SolidQueue::Process.where(id: ids).select(:id, :name, :hostname, :pid, :kind).index_by(&:id)
+      end
+
+      # SUBSTR(col, 1, n) is the one truncation spelling SQLite, PostgreSQL and
+      # MySQL all agree on, so no adapter switch is needed here.
+      def argument_previews(job_ids)
+        return {} if job_ids.empty?
+
+        SolidQueue::Job
+          .where(id: job_ids)
+          .pluck(:id, Arel.sql("SUBSTR(arguments, 1, #{ARGUMENTS_PREVIEW_BYTES})"))
+          .to_h
+      end
+
+      def error_previews(execution_ids)
+        return {} if execution_ids.empty?
+
+        SolidQueue::FailedExecution
+          .where(id: execution_ids)
+          .pluck(:id, Arel.sql("SUBSTR(error, 1, #{ERROR_PREVIEW_BYTES})"))
+          .to_h { |id, raw| [ id, ErrorSummary.parse(raw) ] }
+      end
+
+      def apply_filters(relation)
+        relation = relation.where.not(finished_at: nil) if state == :finished
+        relation = relation.joins(:job) if join_jobs?
+
+        relation = relation.where(jobs_table[:class_name].eq(class_name)) if class_name
+        relation = relation.where(queue_table[:queue_name].eq(queue_name)) if queue_name
+        relation = relation.where(jobs_table[:class_name].matches("#{escape_like(q)}%", nil, true)) if q
+
+        relation
+      end
+
+      def join_jobs?
+        return false unless execution_backed?
+
+        class_name.present? || q.present? || (queue_name.present? && !own_queue_column?)
+      end
+
+      def own_queue_column?
+        model.column_names.include?("queue_name")
+      end
+
+      def jobs_table = SolidQueue::Job.arel_table
+
+      def queue_table
+        own_queue_column? ? model.arel_table : jobs_table
+      end
+
+      def escape_like(value)
+        ActiveRecord::Base.sanitize_sql_like(value.to_s)
+      end
+  end
+end

data/app/models/flightdeck/metrics/bar_chart.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  module Metrics
+    # Succeeded stacked under failed, per bucket.
+    class BarChart < Chart
+      BAR_GAP = 2
+      CORNER = 1.5
+
+      def bars
+        points.each_with_index.map do |point, index|
+          x = plot_left + (slot_width * index)
+          bar_width = [ slot_width - BAR_GAP, 1 ].max
+
+          failed_height = height_for(point.failed)
+          succeeded_height = height_for(point.succeeded)
+          failed_y = plot_bottom - failed_height - succeeded_height
+
+          {
+            x: round(x + (BAR_GAP / 2.0)),
+            width: round(bar_width),
+            corner: CORNER,
+            succeeded: { y: round(plot_bottom - succeeded_height), height: round(succeeded_height) },
+            failed: { y: round(failed_y), height: round(failed_height) },
+            empty: point.total.zero?,
+            title: bar_title(point)
+          }
+        end
+      end
+
+      private
+        def raw_max = points.map(&:total).max.to_i
+
+        def height_for(value)
+          return 0 if value.to_i.zero? || axis_max.zero?
+
+          # Anything non-zero gets at least a hairline, so a bucket with a
+          # single job is visibly different from a bucket with none.
+          [ (value.to_f / axis_max) * plot_height, 1.0 ].max
+        end
+
+        def bar_title(point)
+          "#{full_time(point.at)} · #{number(point.succeeded)} succeeded · #{number(point.failed)} failed"
+        end
+
+        def number(value) = ActiveSupport::NumberHelper.number_to_delimited(value.to_i)
+    end
+  end
+end

data/app/models/flightdeck/metrics/chart.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Flightdeck
+  module Metrics
+    # Geometry for the server-rendered SVG charts.
+    #
+    # All the arithmetic lives here so the ERB partials stay readable: they
+    # place elements and nothing else. Colours are never computed here either —
+    # the partials reference the CSS custom properties, so charts re-theme with
+    # the rest of the page and there is no inline colour to get out of step.
+    class Chart
+      WIDTH = 720
+      HEIGHT = 190
+      PAD_LEFT = 40
+      PAD_RIGHT = 10
+      PAD_TOP = 12
+      PAD_BOTTOM = 22
+      GRID_INTERVALS = 3
+
+      attr_reader :points, :window
+
+      def initialize(points:, window:)
+        @points = points
+        @window = window
+      end
+
+      def width = WIDTH
+      def height = HEIGHT
+      def plot_left = PAD_LEFT
+      def plot_right = WIDTH - PAD_RIGHT
+      def plot_top = PAD_TOP
+      def plot_bottom = HEIGHT - PAD_BOTTOM
+      def plot_width = plot_right - plot_left
+      def plot_height = plot_bottom - plot_top
+      def empty? = points.empty?
+
+      def gridlines
+        (0..GRID_INTERVALS).map do |i|
+          value = axis_max * (GRID_INTERVALS - i) / GRID_INTERVALS.to_f
+          { y: round(plot_top + (plot_height * i / GRID_INTERVALS.to_f)), value: value, label: format_value(value) }
+        end
+      end
+
+      # Ticks are placed on bucket boundaries so a label always names a bucket
+      # that is actually on screen.
+      def x_ticks
+        every = [ window.tick_every.to_i, 1 ].max
+
+        points.each_with_index.filter_map do |point, index|
+          next unless (index % every).zero?
+
+          { x: round(tick_x(index)), label: format_time(point.at, window.tick_format) }
+        end
+      end
+
+      def axis_max
+        @axis_max ||= nice_ceiling(raw_max)
+      end
+
+      def format_time(time, pattern)
+        time.in_time_zone(Flightdeck.config.display_timezone).strftime(pattern)
+      end
+
+      def full_time(time)
+        format_time(time, "%Y-%m-%d %H:%M")
+      end
+
+      private
+        def raw_max = 0
+
+        def tick_x(index)
+          plot_left + (slot_width * index) + (slot_width / 2.0)
+        end
+
+        def slot_width
+          @slot_width ||= points.empty? ? plot_width : plot_width / points.size.to_f
+        end
+
+        def y_for(value)
+          return plot_bottom if axis_max.zero?
+
+          plot_bottom - ((value.to_f / axis_max) * plot_height)
+        end
+
+        # A round number at or above the data's peak, so gridline labels are
+        # readable numbers rather than whatever the maximum happened to be.
+        def nice_ceiling(value)
+          return 1 if value.nil? || value <= 0
+
+          magnitude = 10**Math.log10(value).floor
+          [ 1, 2, 2.5, 5, 10 ].each do |factor|
+            candidate = magnitude * factor
+            return candidate >= 10 ? candidate.round : candidate if candidate >= value
+          end
+          value
+        end
+
+        def format_value(value)
+          case value
+          when 0 then "0"
+          when 0...1 then value.round(2).to_s
+          when 1...1_000 then value < 10 ? value.round(1).to_s.sub(/\.0\z/, "") : value.round.to_s
+          when 1_000...1_000_000 then "#{(value / 1_000.0).round(1).to_s.sub(/\.0\z/, "")}k"
+          else "#{(value / 1_000_000.0).round(1).to_s.sub(/\.0\z/, "")}M"
+          end
+        end
+
+        def round(value) = value.round(2)
+    end
+  end
+end