flightdeck 0.5.3 → 0.6.0

data/app/models/flightdeck/error_summary.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Flightdeck
-  # A failed execution's `error` column is JSON that can run to many kilobytes
-  # once a backtrace is in it. List views only ever read a bounded prefix of it,
-  # which means the JSON is usually truncated mid-string — so parse when we can
-  # and fall back to scanning for the two fields we need.
-  class ErrorSummary
-    UNKNOWN = "UnknownError"
-
-    attr_reader :exception_class, :message
-
-    def self.parse(raw)
-      new(raw)
-    end
-
-    def self.none
-      new(nil)
-    end
-
-    def initialize(raw)
-      @exception_class, @message = extract(raw)
-    end
-
-    def present?
-      exception_class.present?
-    end
-
-    def group_key
-      exception_class.presence || UNKNOWN
-    end
-
-    def to_s
-      return "" unless present?
-
-      message.present? ? "#{exception_class}: #{message}" : exception_class
-    end
-
-    private
-      def extract(raw)
-        return [ nil, nil ] if raw.blank?
-
-        parsed = safe_parse(raw)
-        return [ parsed["exception_class"].presence, parsed["message"].to_s ] if parsed.is_a?(Hash)
-
-        [
-          raw[/"exception_class"\s*:\s*"((?:[^"\\]|\\.)*)"/, 1]&.then { |value| unescape(value) },
-          raw[/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/, 1]&.then { |value| unescape(value) }.to_s
-        ]
-      end
-
-      def safe_parse(raw)
-        JSON.parse(raw)
-      rescue JSON::ParserError
-        nil
-      end
-
-      def unescape(value)
-        JSON.parse(%("#{value}"))
-      rescue JSON::ParserError
-        value
-      end
-  end
-end

data/app/models/flightdeck/grouped_failures.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Flightdeck
-  # Groups the failed rows of the *current page* by exception class.
-  #
-  # Grouping happens in Ruby on purpose. Extracting the exception class in SQL
-  # would mean a JSON function per adapter (and an unindexed scan of a large
-  # text column either way), which is a bad trade for a header row.
-  class GroupedFailures
-    Group = Struct.new(:key, :rows, keyword_init: true) do
-      def count = rows.size
-      def job_ids = rows.map(&:id)
-      def first_failed_at = rows.filter_map(&:failed_at).min
-      def last_failed_at = rows.filter_map(&:failed_at).max
-    end
-
-    def self.build(rows)
-      rows
-        .group_by { |row| row.error_summary.group_key }
-        .map { |key, grouped| Group.new(key: key, rows: grouped) }
-        .sort_by { |group| [ -group.count, group.key ] }
-    end
-  end
-end

data/app/models/flightdeck/job_detail.rb

@@ -1,156 +0,0 @@
-# frozen_string_literal: true
-
-module Flightdeck
-  # Everything the job show page needs, assembled from Solid Queue's own models.
-  #
-  # Arguments are shown exactly as they were stored. They are never handed to
-  # ActiveJob for deserialization: that would load host application classes (and
-  # run their code) just to render a dashboard.
-  class JobDetail
-    ARGUMENTS_DISPLAY_LIMIT = 10.kilobytes
-
-    # A frame belongs to the application unless it lives in an installed gem or
-    # in Ruby's own library directories.
-    VENDOR_FRAME = %r{
-      (?:\A|/)(?:gems|ruby|rubygems|bundler|vendor/bundle)/ |
-      \A<internal: | \A/usr/lib/ruby/
-    }x
-
-    attr_reader :job, :state, :execution, :process
-
-    def self.find(id)
-      job = SolidQueue::Job.find(id)
-      annotation = JobRow.annotate([ job.id ])[job.id]
-
-      new(job: job, annotation: annotation)
-    end
-
-    def initialize(job:, annotation: nil)
-      @job = job
-      @state = JobRow.state_for(job, annotation)
-      @execution = load_execution(annotation)
-      @process = load_process
-    end
-
-    def id = job.id
-    def class_name = job.class_name
-    def queue_name = job.queue_name
-    def priority = job.priority
-    def active_job_id = job.active_job_id
-    def concurrency_key = job.concurrency_key
-    def enqueued_at = job.created_at
-    def scheduled_at = job.scheduled_at
-    def finished_at = job.finished_at
-
-    def failed? = state == :failed
-    def retryable? = failed?
-    def discardable? = state != :in_progress && state != :finished
-
-    def failed_execution
-      execution if failed?
-    end
-
-    # --- arguments ----------------------------------------------------------
-
-    # `job.arguments` is already a Ruby structure (Solid Queue stores it with a
-    # JSON coder); pretty-printing it is a formatting step, not deserialization.
-    def arguments_json
-      @arguments_json ||= begin
-        JSON.pretty_generate(job.arguments)
-      rescue StandardError
-        job.arguments.to_s
-      end
-    end
-
-    def arguments_truncated?
-      arguments_json.bytesize > ARGUMENTS_DISPLAY_LIMIT
-    end
-
-    def arguments_preview
-      return arguments_json unless arguments_truncated?
-
-      arguments_json.byteslice(0, ARGUMENTS_DISPLAY_LIMIT).scrub("")
-    end
-
-    def arguments_bytes = arguments_json.bytesize
-
-    # --- error --------------------------------------------------------------
-
-    def error_class
-      failed_execution&.exception_class
-    end
-
-    def error_message
-      failed_execution&.message
-    end
-
-    def backtrace
-      @backtrace ||= Array(failed_execution&.backtrace).first(Flightdeck.config.backtrace_lines)
-    end
-
-    def backtrace_total = Array(failed_execution&.backtrace).size
-
-    def backtrace_truncated? = backtrace_total > backtrace.size
-
-    def backtrace_frames
-      backtrace.map { |line| { line: line, app: app_frame?(line) } }
-    end
-
-    def app_frame?(line)
-      !VENDOR_FRAME.match?(line.to_s)
-    end
-
-    # --- timeline -----------------------------------------------------------
-
-    Event = Struct.new(:label, :at, :detail, :status, keyword_init: true)
-
-    # Built only from timestamps Solid Queue actually keeps. Steps whose row has
-    # already been consumed (a finished job's claim, say) are simply absent
-    # rather than invented.
-    def timeline
-      events = [ Event.new(label: "Enqueued", at: enqueued_at, detail: "queue #{queue_name}", status: :ok) ]
-
-      if scheduled_at.present? && scheduled_at > enqueued_at
-        events << Event.new(label: "Scheduled", at: scheduled_at,
-                            detail: scheduled_at.future? ? "not yet due" : "became due", status: :ok)
-      end
-
-      case state
-      when :ready
-        events << Event.new(label: "Ready", at: execution&.created_at, detail: "waiting for a worker", status: :now)
-      when :blocked
-        events << Event.new(label: "Blocked", at: execution&.created_at,
-                            detail: "concurrency key #{job.concurrency_key}", status: :now)
-      when :in_progress
-        events << Event.new(label: "Claimed", at: execution&.created_at, detail: process_label, status: :now)
-      when :failed
-        events << Event.new(label: "Failed", at: execution&.created_at, detail: error_class, status: :bad)
-        events << Event.new(label: "Awaiting decision", at: nil,
-                            detail: "retry or discard", status: :now)
-      end
-
-      events << Event.new(label: "Finished", at: finished_at, detail: nil, status: :ok) if finished_at.present?
-      events.select { |event| event.at.present? || event.status == :now }
-    end
-
-    def process_label
-      return nil unless process
-
-      [ process.name, ("pid #{process.pid}" if process.pid) ].compact.join(" · ")
-    end
-
-    private
-      def load_execution(annotation)
-        return nil unless annotation
-
-        model = JobRow.model_for(state)
-        model&.find_by(id: annotation[:execution_id])
-      end
-
-      def load_process
-        return nil unless execution.respond_to?(:process_id) && execution.process_id
-
-        SolidQueue::Process.find_by(id: execution.process_id)
-      end
-  end
-end

data/app/models/flightdeck/job_row.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module Flightdeck
-  # One row of a jobs list. Deliberately dumb: JobsQuery assembles these from
-  # bounded queries and the view asks them formatting questions.
-  class JobRow
-    # Every table a job can be sitting in, in the order we report them. A job is
-    # only ever in one of these at a time, but we probe all five so a row is
-    # never silently mislabelled during a transition.
-    EXECUTION_MODELS = {
-      ready: "SolidQueue::ReadyExecution",
-      scheduled: "SolidQueue::ScheduledExecution",
-      in_progress: "SolidQueue::ClaimedExecution",
-      blocked: "SolidQueue::BlockedExecution",
-      failed: "SolidQueue::FailedExecution"
-    }.freeze
-
-    class << self
-      def model_for(state)
-        name = EXECUTION_MODELS[state.to_sym]
-        name && name.constantize
-      end
-
-      # How a job's state is derived, in one place: `finished_at` wins, then
-      # whichever execution table `annotate` found the job in, then :unknown
-      # for the transition window where a job is between tables.
-      def state_for(job, annotation)
-        return :finished if job.finished_at.present?
-        return annotation[:state] if annotation
-
-        :unknown
-      end
-
-      # Resolves the state of an already-loaded page of jobs with one
-      # `WHERE job_id IN (page_ids)` lookup per executions table. Returns
-      # { job_id => { state:, execution_id:, process_id: } }.
-      def annotate(job_ids)
-        job_ids = Array(job_ids).compact.uniq
-        return {} if job_ids.empty?
-
-        EXECUTION_MODELS.each_with_object({}) do |(state, model_name), found|
-          model = model_name.constantize
-          columns = [ :id, :job_id ]
-          columns << :process_id if model.column_names.include?("process_id")
-
-          model.where(job_id: job_ids).pluck(*columns).each do |values|
-            job_id = values[1]
-            next if found.key?(job_id)
-
-            found[job_id] = { state: state, execution_id: values[0], process_id: values[2] }
-          end
-        end
-      end
-    end
-
-    attr_reader :job, :state, :execution, :args_preview, :error_summary, :process
-
-    def initialize(job:, state:, execution: nil, args_preview: nil, error_summary: nil, process: nil)
-      @job = job
-      @state = state
-      @execution = execution
-      @args_preview = args_preview
-      @error_summary = error_summary || ErrorSummary.none
-      @process = process
-    end
-
-    def id = job.id
-    def class_name = job.class_name
-    def queue_name = job.queue_name
-    def priority = job.priority
-    def active_job_id = job.active_job_id
-    def concurrency_key = job.concurrency_key
-    def enqueued_at = job.created_at
-    def scheduled_at = job.scheduled_at
-    def finished_at = job.finished_at
-
-    def failed? = state == :failed
-    def discardable? = state != :in_progress
-
-    # The moment the job first became eligible to run — what "waiting" is
-    # measured from, so a job scheduled for next week is not reported as having
-    # waited a week.
-    def due_at
-      scheduled_at || enqueued_at
-    end
-
-    def execution_started_at
-      execution&.created_at if state == :in_progress
-    end
-
-    def failed_at
-      execution&.created_at if failed?
-    end
-
-    # Attempt number, when ActiveJob recorded one in the serialized payload. We
-    # only ever see a truncated prefix of the arguments, so this is best-effort
-    # by design and returns nil rather than guessing.
-    def attempts
-      value = args_preview.to_s[/"executions"\s*:\s*(\d+)/, 1]
-      value && value.to_i + 1
-    end
-  end
-end

data/app/models/flightdeck/jobs_query.rb

@@ -1,233 +0,0 @@
-# frozen_string_literal: true
-
-module Flightdeck
-  # Builds every jobs list in Flightdeck.
-  #
-  # Rule zero: everything goes through SolidQueue's own models, so multi-database
-  # routing comes for free and the host's connection is never touched.
-  #
-  # Two shapes of query, chosen by state:
-  #   * an execution-backed state drives from its own table (that is what makes
-  #     the state true) and then loads the matching jobs;
-  #   * "all" and "finished" drive from solid_queue_jobs and have their state
-  #     annotated afterwards.
-  #
-  # The `arguments` and `error` columns are never in a list's driving query.
-  # Both are fetched afterwards for the current page only, and truncated in SQL
-  # so a page of pathological payloads still costs a bounded number of bytes.
-  class JobsQuery
-    STATES = %i[all ready scheduled in_progress blocked finished failed].freeze
-
-    # Enough that the ActiveJob envelope (job_class, job_id, queue_name…) is
-    # comfortably past and the arguments themselves are in the prefix.
-    ARGUMENTS_PREVIEW_BYTES = 800
-    ERROR_PREVIEW_BYTES = 1_000
-
-    class InvalidState < ArgumentError; end
-
-    attr_reader :state, :class_name, :queue_name, :q, :before_id, :limit, :count_cap
-
-    def initialize(state: :all, class_name: nil, queue_name: nil, q: nil, before_id: nil,
-                   limit: nil, count_cap: nil)
-      @state = normalize_state(state)
-      @class_name = class_name.presence
-      @queue_name = queue_name.presence
-      @q = q.presence
-      @before_id = before_id.presence && Integer(before_id, exception: false)
-      @limit = (limit || Flightdeck.config.per_page).to_i.clamp(1, 200)
-      @count_cap = (count_cap || Flightdeck.config.count_cap).to_i
-    end
-
-    def rows
-      @rows ||= build_rows
-    end
-
-    def next_cursor
-      rows
-      @next_cursor
-    end
-
-    def next_page? = next_cursor.present?
-
-    # Capped so that a jobs table with tens of millions of rows cannot turn a
-    # page render into a full scan. Views render `count_capped?` as "500,000+".
-    #
-    # Cached for one poll interval: the state tabs and the sidebar issue several
-    # of these per render, and a count that is at most one poll stale is exactly
-    # as fresh as the page it appears on. Mutations wrap themselves in
-    # Cache.bypass so an action's own response never shows pre-action numbers.
-    def count
-      @count ||= Flightdeck::Cache.fetch("count", state, count_cap, cache_filters,
-                                         expires_in: Flightdeck.config.poll_interval) do
-        filtered_relation.limit(count_cap).count
-      end
-    end
-
-    def count_capped? = count >= count_cap
-
-    def filters?
-      class_name.present? || queue_name.present? || q.present?
-    end
-
-    # The filtered relation with no ordering, limit or cursor — what bulk
-    # "apply to all matching" re-runs server-side.
-    def filtered_relation
-      @filtered_relation ||= apply_filters(model.all)
-    end
-
-    def model
-      @model ||= JobRow.model_for(state) || SolidQueue::Job
-    end
-
-    def execution_backed? = model != SolidQueue::Job
-
-    # Counts for the state tabs, each capped independently.
-    def self.state_counts(**filters)
-      STATES.index_with do |state|
-        query = new(**filters.except(:state, :before_id), state: state)
-        { count: query.count, capped: query.count_capped? }
-      end
-    end
-
-    private
-      # The columns list queries must never drag along: `arguments` on jobs and
-      # `error` on failed_executions can each run to kilobytes per row.
-      PAYLOAD_COLUMNS = %w[arguments error].freeze
-
-      # Only the filters change what a count means — the cursor and page size do
-      # not, so they are deliberately absent from the key.
-      def cache_filters
-        { class_name: class_name, queue_name: queue_name, q: q }
-      end
-
-      def normalize_state(value)
-        state = value.presence&.to_sym || :all
-        raise InvalidState, "unknown job state #{value.inspect}" unless STATES.include?(state)
-
-        state
-      end
-
-      def build_rows
-        page = paginated_records
-        @next_cursor = page.size > limit ? page[limit - 1].id : nil
-        page = page.first(limit)
-
-        return [] if page.empty?
-
-        execution_backed? ? rows_from_executions(page) : rows_from_jobs(page)
-      end
-
-      def paginated_records
-        relation = filtered_relation
-          .reorder(model.arel_table[:id].desc)
-          .limit(limit + 1)
-
-        relation = relation.where(model.arel_table[:id].lt(before_id)) if before_id
-        relation.select(driving_columns).to_a
-      end
-
-      # Never `SELECT *`: it would drag a payload column through every list
-      # query.
-      def driving_columns
-        (model.column_names - PAYLOAD_COLUMNS).map { |name| model.arel_table[name] }
-      end
-
-      def rows_from_executions(executions)
-        job_ids = executions.map(&:job_id)
-        jobs = jobs_by_id(job_ids)
-        previews = argument_previews(job_ids)
-        errors = state == :failed ? error_previews(executions.map(&:id)) : {}
-        processes = state == :in_progress ? processes_by_id(executions.map(&:process_id)) : {}
-
-        executions.filter_map do |execution|
-          job = jobs[execution.job_id]
-          next unless job
-
-          JobRow.new(
-            job: job,
-            state: state,
-            execution: execution,
-            args_preview: previews[job.id],
-            error_summary: errors[execution.id],
-            process: processes[execution.try(:process_id)]
-          )
-        end
-      end
-
-      def rows_from_jobs(jobs)
-        job_ids = jobs.map(&:id)
-        previews = argument_previews(job_ids)
-        annotations = state == :finished ? {} : JobRow.annotate(job_ids)
-
-        jobs.map do |job|
-          JobRow.new(
-            job: job,
-            state: JobRow.state_for(job, annotations[job.id]),
-            args_preview: previews[job.id]
-          )
-        end
-      end
-
-      def jobs_by_id(job_ids)
-        SolidQueue::Job.where(id: job_ids).select(SolidQueue::Job.column_names - PAYLOAD_COLUMNS).index_by(&:id)
-      end
-
-      def processes_by_id(process_ids)
-        ids = process_ids.compact.uniq
-        return {} if ids.empty?
-
-        SolidQueue::Process.where(id: ids).select(:id, :name, :hostname, :pid, :kind).index_by(&:id)
-      end
-
-      # SUBSTR(col, 1, n) is the one truncation spelling SQLite, PostgreSQL and
-      # MySQL all agree on, so no adapter switch is needed here.
-      def argument_previews(job_ids)
-        return {} if job_ids.empty?
-
-        SolidQueue::Job
-          .where(id: job_ids)
-          .pluck(:id, Arel.sql("SUBSTR(arguments, 1, #{ARGUMENTS_PREVIEW_BYTES})"))
-          .to_h
-      end
-
-      def error_previews(execution_ids)
-        return {} if execution_ids.empty?
-
-        SolidQueue::FailedExecution
-          .where(id: execution_ids)
-          .pluck(:id, Arel.sql("SUBSTR(error, 1, #{ERROR_PREVIEW_BYTES})"))
-          .to_h { |id, raw| [ id, ErrorSummary.parse(raw) ] }
-      end
-
-      def apply_filters(relation)
-        relation = relation.where.not(finished_at: nil) if state == :finished
-        relation = relation.joins(:job) if join_jobs?
-
-        relation = relation.where(jobs_table[:class_name].eq(class_name)) if class_name
-        relation = relation.where(queue_table[:queue_name].eq(queue_name)) if queue_name
-        relation = relation.where(jobs_table[:class_name].matches("#{escape_like(q)}%", nil, true)) if q
-
-        relation
-      end
-
-      def join_jobs?
-        return false unless execution_backed?
-
-        class_name.present? || q.present? || (queue_name.present? && !own_queue_column?)
-      end
-
-      def own_queue_column?
-        model.column_names.include?("queue_name")
-      end
-
-      def jobs_table = SolidQueue::Job.arel_table
-
-      def queue_table
-        own_queue_column? ? model.arel_table : jobs_table
-      end
-
-      def escape_like(value)
-        ActiveRecord::Base.sanitize_sql_like(value.to_s)
-      end
-  end
-end

data/app/models/flightdeck/metrics/bar_chart.rb

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module Flightdeck
-  module Metrics
-    # Succeeded stacked under failed, per bucket.
-    class BarChart < Chart
-      BAR_GAP = 2
-      CORNER = 1.5
-
-      def bars
-        points.each_with_index.map do |point, index|
-          x = plot_left + (slot_width * index)
-          bar_width = [ slot_width - BAR_GAP, 1 ].max
-
-          failed_height = height_for(point.failed)
-          succeeded_height = height_for(point.succeeded)
-          failed_y = plot_bottom - failed_height - succeeded_height
-
-          {
-            x: round(x + (BAR_GAP / 2.0)),
-            width: round(bar_width),
-            corner: CORNER,
-            succeeded: { y: round(plot_bottom - succeeded_height), height: round(succeeded_height) },
-            failed: { y: round(failed_y), height: round(failed_height) },
-            empty: point.total.zero?,
-            title: bar_title(point)
-          }
-        end
-      end
-
-      private
-        def raw_max = points.map(&:total).max.to_i
-
-        def height_for(value)
-          return 0 if value.to_i.zero? || axis_max.zero?
-
-          # Anything non-zero gets at least a hairline, so a bucket with a
-          # single job is visibly different from a bucket with none.
-          [ (value.to_f / axis_max) * plot_height, 1.0 ].max
-        end
-
-        def bar_title(point)
-          "#{full_time(point.at)} · #{number(point.succeeded)} succeeded · #{number(point.failed)} failed"
-        end
-
-        def number(value) = ActiveSupport::NumberHelper.number_to_delimited(value.to_i)
-    end
-  end
-end