flightdeck 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -199
- metadata +12 -96
- data/MIT-LICENSE +0 -21
- data/app/assets/flightdeck/flightdeck-3967b9e6c506.css +0 -3
- data/app/assets/flightdeck/flightdeck-3aac23f80afe.js +0 -414
- data/app/assets/flightdeck/manifest.json +0 -16
- data/app/controllers/concerns/flightdeck/toasts.rb +0 -35
- data/app/controllers/flightdeck/application_controller.rb +0 -110
- data/app/controllers/flightdeck/assets_controller.rb +0 -39
- data/app/controllers/flightdeck/jobs/actions_controller.rb +0 -166
- data/app/controllers/flightdeck/jobs/discards_controller.rb +0 -43
- data/app/controllers/flightdeck/jobs/retries_controller.rb +0 -22
- data/app/controllers/flightdeck/jobs_controller.rb +0 -39
- data/app/controllers/flightdeck/overview_controller.rb +0 -11
- data/app/controllers/flightdeck/processes_controller.rb +0 -54
- data/app/controllers/flightdeck/queues_controller.rb +0 -56
- data/app/controllers/flightdeck/recurring_tasks_controller.rb +0 -51
- data/app/helpers/flightdeck/application_helper.rb +0 -85
- data/app/helpers/flightdeck/infrastructure_helper.rb +0 -24
- data/app/helpers/flightdeck/jobs_helper.rb +0 -117
- data/app/helpers/flightdeck/metrics_helper.rb +0 -49
- data/app/models/flightdeck/arguments_preview.rb +0 -95
- data/app/models/flightdeck/bulk_action.rb +0 -83
- data/app/models/flightdeck/cache.rb +0 -51
- data/app/models/flightdeck/cron_schedule.rb +0 -102
- data/app/models/flightdeck/duration.rb +0 -23
- data/app/models/flightdeck/error_summary.rb +0 -64
- data/app/models/flightdeck/grouped_failures.rb +0 -24
- data/app/models/flightdeck/job_detail.rb +0 -156
- data/app/models/flightdeck/job_row.rb +0 -103
- data/app/models/flightdeck/jobs_query.rb +0 -233
- data/app/models/flightdeck/metrics/bar_chart.rb +0 -49
- data/app/models/flightdeck/metrics/chart.rb +0 -111
- data/app/models/flightdeck/metrics/line_chart.rb +0 -63
- data/app/models/flightdeck/metrics/series.rb +0 -165
- data/app/models/flightdeck/metrics/sparkline.rb +0 -45
- data/app/models/flightdeck/metrics/time_bucket.rb +0 -95
- data/app/models/flightdeck/overview.rb +0 -210
- data/app/models/flightdeck/process_registry.rb +0 -102
- data/app/models/flightdeck/queue_stats.rb +0 -86
- data/app/models/flightdeck/recurring_catalog.rb +0 -114
- data/app/views/flightdeck/jobs/_failed_row.html.erb +0 -33
- data/app/views/flightdeck/jobs/_filter_bar.html.erb +0 -29
- data/app/views/flightdeck/jobs/_list.html.erb +0 -94
- data/app/views/flightdeck/jobs/_pager.html.erb +0 -22
- data/app/views/flightdeck/jobs/_row.html.erb +0 -23
- data/app/views/flightdeck/jobs/_state_tabs.html.erb +0 -10
- data/app/views/flightdeck/jobs/index.html.erb +0 -10
- data/app/views/flightdeck/jobs/show.html.erb +0 -100
- data/app/views/flightdeck/metrics/_bar_chart.html.erb +0 -34
- data/app/views/flightdeck/metrics/_line_chart.html.erb +0 -38
- data/app/views/flightdeck/metrics/_range_control.html.erb +0 -7
- data/app/views/flightdeck/metrics/_sparkline.html.erb +0 -9
- data/app/views/flightdeck/overview/_completion.html.erb +0 -19
- data/app/views/flightdeck/overview/_failures.html.erb +0 -28
- data/app/views/flightdeck/overview/_fleet.html.erb +0 -21
- data/app/views/flightdeck/overview/_queues.html.erb +0 -41
- data/app/views/flightdeck/overview/_throughput.html.erb +0 -23
- data/app/views/flightdeck/overview/_tiles.html.erb +0 -14
- data/app/views/flightdeck/overview/index.html.erb +0 -29
- data/app/views/flightdeck/processes/_fleet.html.erb +0 -54
- data/app/views/flightdeck/processes/_row.html.erb +0 -28
- data/app/views/flightdeck/processes/index.html.erb +0 -5
- data/app/views/flightdeck/queues/_cards.html.erb +0 -46
- data/app/views/flightdeck/queues/index.html.erb +0 -5
- data/app/views/flightdeck/recurring_tasks/_table.html.erb +0 -53
- data/app/views/flightdeck/recurring_tasks/index.html.erb +0 -5
- data/app/views/flightdeck/shared/_toast.html.erb +0 -7
- data/app/views/flightdeck/shared/refresh.turbo_stream.erb +0 -14
- data/app/views/layouts/flightdeck/application.html.erb +0 -110
- data/config/routes.rb +0 -40
- data/lib/flightdeck/assets.rb +0 -72
- data/lib/flightdeck/configuration.rb +0 -82
- data/lib/flightdeck/engine.rb +0 -31
- data/lib/flightdeck/version.rb +0 -5
- data/lib/flightdeck.rb +0 -31
|
@@ -1,111 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Flightdeck
|
|
4
|
-
module Metrics
|
|
5
|
-
# Geometry for the server-rendered SVG charts.
|
|
6
|
-
#
|
|
7
|
-
# All the arithmetic lives here so the ERB partials stay readable: they
|
|
8
|
-
# place elements and nothing else. Colours are never computed here either —
|
|
9
|
-
# the partials reference the CSS custom properties, so charts re-theme with
|
|
10
|
-
# the rest of the page and there is no inline colour to get out of step.
|
|
11
|
-
class Chart
|
|
12
|
-
WIDTH = 720
|
|
13
|
-
HEIGHT = 190
|
|
14
|
-
PAD_LEFT = 40
|
|
15
|
-
PAD_RIGHT = 10
|
|
16
|
-
PAD_TOP = 12
|
|
17
|
-
PAD_BOTTOM = 22
|
|
18
|
-
GRID_INTERVALS = 3
|
|
19
|
-
|
|
20
|
-
attr_reader :points, :window
|
|
21
|
-
|
|
22
|
-
def initialize(points:, window:)
|
|
23
|
-
@points = points
|
|
24
|
-
@window = window
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def width = WIDTH
|
|
28
|
-
def height = HEIGHT
|
|
29
|
-
def plot_left = PAD_LEFT
|
|
30
|
-
def plot_right = WIDTH - PAD_RIGHT
|
|
31
|
-
def plot_top = PAD_TOP
|
|
32
|
-
def plot_bottom = HEIGHT - PAD_BOTTOM
|
|
33
|
-
def plot_width = plot_right - plot_left
|
|
34
|
-
def plot_height = plot_bottom - plot_top
|
|
35
|
-
def empty? = points.empty?
|
|
36
|
-
|
|
37
|
-
def gridlines
|
|
38
|
-
(0..GRID_INTERVALS).map do |i|
|
|
39
|
-
value = axis_max * (GRID_INTERVALS - i) / GRID_INTERVALS.to_f
|
|
40
|
-
{ y: round(plot_top + (plot_height * i / GRID_INTERVALS.to_f)), value: value, label: format_value(value) }
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
# Ticks are placed on bucket boundaries so a label always names a bucket
|
|
45
|
-
# that is actually on screen.
|
|
46
|
-
def x_ticks
|
|
47
|
-
every = [ window.tick_every.to_i, 1 ].max
|
|
48
|
-
|
|
49
|
-
points.each_with_index.filter_map do |point, index|
|
|
50
|
-
next unless (index % every).zero?
|
|
51
|
-
|
|
52
|
-
{ x: round(tick_x(index)), label: format_time(point.at, window.tick_format) }
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
def axis_max
|
|
57
|
-
@axis_max ||= nice_ceiling(raw_max)
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
def format_time(time, pattern)
|
|
61
|
-
time.in_time_zone(Flightdeck.config.display_timezone).strftime(pattern)
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
def full_time(time)
|
|
65
|
-
format_time(time, "%Y-%m-%d %H:%M")
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
private
|
|
69
|
-
def raw_max = 0
|
|
70
|
-
|
|
71
|
-
def tick_x(index)
|
|
72
|
-
plot_left + (slot_width * index) + (slot_width / 2.0)
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
def slot_width
|
|
76
|
-
@slot_width ||= points.empty? ? plot_width : plot_width / points.size.to_f
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
def y_for(value)
|
|
80
|
-
return plot_bottom if axis_max.zero?
|
|
81
|
-
|
|
82
|
-
plot_bottom - ((value.to_f / axis_max) * plot_height)
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
# A round number at or above the data's peak, so gridline labels are
|
|
86
|
-
# readable numbers rather than whatever the maximum happened to be.
|
|
87
|
-
def nice_ceiling(value)
|
|
88
|
-
return 1 if value.nil? || value <= 0
|
|
89
|
-
|
|
90
|
-
magnitude = 10**Math.log10(value).floor
|
|
91
|
-
[ 1, 2, 2.5, 5, 10 ].each do |factor|
|
|
92
|
-
candidate = magnitude * factor
|
|
93
|
-
return candidate >= 10 ? candidate.round : candidate if candidate >= value
|
|
94
|
-
end
|
|
95
|
-
value
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
def format_value(value)
|
|
99
|
-
case value
|
|
100
|
-
when 0 then "0"
|
|
101
|
-
when 0...1 then value.round(2).to_s
|
|
102
|
-
when 1...1_000 then value < 10 ? value.round(1).to_s.sub(/\.0\z/, "") : value.round.to_s
|
|
103
|
-
when 1_000...1_000_000 then "#{(value / 1_000.0).round(1).to_s.sub(/\.0\z/, "")}k"
|
|
104
|
-
else "#{(value / 1_000_000.0).round(1).to_s.sub(/\.0\z/, "")}M"
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def round(value) = value.round(2)
|
|
109
|
-
end
|
|
110
|
-
end
|
|
111
|
-
end
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Flightdeck
|
|
4
|
-
module Metrics
|
|
5
|
-
# Average completion time: area under a line, with gaps where no jobs
|
|
6
|
-
# finished at all.
|
|
7
|
-
class LineChart < Chart
|
|
8
|
-
DOT_RADIUS = 3
|
|
9
|
-
|
|
10
|
-
def coordinates
|
|
11
|
-
@coordinates ||= points.each_with_index.map do |point, index|
|
|
12
|
-
next nil if point.seconds.nil?
|
|
13
|
-
|
|
14
|
-
{ x: round(plot_left + (slot_width * index) + (slot_width / 2.0)),
|
|
15
|
-
y: round(y_for(point.seconds)),
|
|
16
|
-
seconds: point.seconds,
|
|
17
|
-
title: point_title(point) }
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
# Contiguous runs of buckets that had data. Drawing them separately keeps
|
|
22
|
-
# a gap in the data looking like a gap, instead of a straight line across
|
|
23
|
-
# a period we know nothing about.
|
|
24
|
-
def segments
|
|
25
|
-
coordinates.chunk_while { |a, b| !a.nil? && !b.nil? }.filter_map do |chunk|
|
|
26
|
-
present = chunk.compact
|
|
27
|
-
present.size.positive? ? present : nil
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
def line_path(segment)
|
|
32
|
-
return "" if segment.empty?
|
|
33
|
-
|
|
34
|
-
segment.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def area_path(segment)
|
|
38
|
-
return "" if segment.empty?
|
|
39
|
-
|
|
40
|
-
"#{line_path(segment)} L #{segment.last[:x]} #{plot_bottom} L #{segment.first[:x]} #{plot_bottom} Z"
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
def markers = coordinates.compact
|
|
44
|
-
|
|
45
|
-
def last_marker = markers.last
|
|
46
|
-
|
|
47
|
-
def any_data? = markers.any?
|
|
48
|
-
|
|
49
|
-
private
|
|
50
|
-
def raw_max = points.filter_map(&:seconds).max.to_f
|
|
51
|
-
|
|
52
|
-
# The zero gridline reads "0s" rather than "0ms": it is an axis origin,
|
|
53
|
-
# not a measurement.
|
|
54
|
-
def format_value(value)
|
|
55
|
-
value.zero? ? "0s" : Flightdeck::Duration.humanize(value)
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def point_title(point)
|
|
59
|
-
"#{full_time(point.at)} · avg #{format_value(point.seconds)} to completion"
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
end
|
|
63
|
-
end
|
|
@@ -1,165 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Flightdeck
|
|
4
|
-
module Metrics
|
|
5
|
-
# Time series for the Overview charts, computed live from the Solid Queue
|
|
6
|
-
# tables — no rollup tables, no background aggregation.
|
|
7
|
-
#
|
|
8
|
-
# Every series is gap-filled in Ruby so a chart always has a complete axis:
|
|
9
|
-
# a bucket with no rows is a real zero (or a real "no data"), not a missing
|
|
10
|
-
# point that would make the chart lie about its own shape.
|
|
11
|
-
class Series
|
|
12
|
-
Window = Struct.new(:key, :label, :duration, :bucket_seconds, :subtitle, :tick_every,
|
|
13
|
-
:tick_format, keyword_init: true) do
|
|
14
|
-
def bucket_count = (duration / bucket_seconds).to_i
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
WINDOWS = {
|
|
18
|
-
"1h" => Window.new(key: "1h", label: "1H", duration: 1.hour, bucket_seconds: 5.minutes.to_i,
|
|
19
|
-
subtitle: "per 5 minutes", tick_every: 3, tick_format: "%H:%M"),
|
|
20
|
-
"24h" => Window.new(key: "24h", label: "24H", duration: 24.hours, bucket_seconds: 1.hour.to_i,
|
|
21
|
-
subtitle: "per hour", tick_every: 4, tick_format: "%H:%M"),
|
|
22
|
-
"7d" => Window.new(key: "7d", label: "7D", duration: 7.days, bucket_seconds: 6.hours.to_i,
|
|
23
|
-
subtitle: "per 6 hours", tick_every: 4, tick_format: "%-m/%-d")
|
|
24
|
-
}.freeze
|
|
25
|
-
|
|
26
|
-
DEFAULT_WINDOW = "24h"
|
|
27
|
-
SPARKLINE_HOURS = 16
|
|
28
|
-
SPARKLINE_BUCKET = 1.hour.to_i
|
|
29
|
-
|
|
30
|
-
Point = Struct.new(:at, :succeeded, :failed, :seconds, keyword_init: true) do
|
|
31
|
-
def total = succeeded.to_i + failed.to_i
|
|
32
|
-
def blank? = seconds.nil?
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
attr_reader :window, :now
|
|
36
|
-
|
|
37
|
-
def self.window_for(key)
|
|
38
|
-
WINDOWS.fetch(key.to_s, WINDOWS.fetch(DEFAULT_WINDOW))
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
def initialize(window: DEFAULT_WINDOW, now: Time.current)
|
|
42
|
-
@window = window.is_a?(Window) ? window : self.class.window_for(window)
|
|
43
|
-
@now = now.utc
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
# Bucket start times, aligned to absolute epoch multiples so that the same
|
|
47
|
-
# bucket has the same boundaries in every process and every request.
|
|
48
|
-
def buckets
|
|
49
|
-
@buckets ||= begin
|
|
50
|
-
last = align(@now.to_i)
|
|
51
|
-
count = window.bucket_count
|
|
52
|
-
Array.new(count) { |i| last - ((count - 1 - i) * window.bucket_seconds) }
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
def starts_at = Time.at(buckets.first).utc
|
|
57
|
-
def ends_at = Time.at(buckets.last + window.bucket_seconds).utc
|
|
58
|
-
|
|
59
|
-
# Jobs finished (succeeded) and executions failed, per bucket.
|
|
60
|
-
#
|
|
61
|
-
# A job that failed has no finished_at — Solid Queue leaves it null and
|
|
62
|
-
# records a failed_executions row — so the two series never double count.
|
|
63
|
-
def throughput
|
|
64
|
-
cached(:throughput) do
|
|
65
|
-
succeeded = grouped_count(SolidQueue::Job.where(finished_at: starts_at..), :finished_at)
|
|
66
|
-
failed = grouped_count(SolidQueue::FailedExecution.where(created_at: starts_at..), :created_at,
|
|
67
|
-
table: SolidQueue::FailedExecution.table_name)
|
|
68
|
-
|
|
69
|
-
buckets.map do |bucket|
|
|
70
|
-
Point.new(at: Time.at(bucket).utc,
|
|
71
|
-
succeeded: succeeded.fetch(bucket, 0),
|
|
72
|
-
failed: failed.fetch(bucket, 0))
|
|
73
|
-
end
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
# Average wall-clock time from enqueue to finish, per bucket.
|
|
78
|
-
#
|
|
79
|
-
# This is time *to completion*, not time to start: Solid Queue deletes the
|
|
80
|
-
# claimed_executions row when a job finishes, so the moment a job actually
|
|
81
|
-
# started is not recoverable after the fact and no honest historical
|
|
82
|
-
# time-to-start series can be built from these tables.
|
|
83
|
-
def completion_time
|
|
84
|
-
cached(:completion_time) do
|
|
85
|
-
averages = SolidQueue::Job
|
|
86
|
-
.where(finished_at: starts_at..)
|
|
87
|
-
.group(TimeBucket.sql("#{jobs_table}.finished_at", window.bucket_seconds))
|
|
88
|
-
.average(TimeBucket.elapsed_sql("#{jobs_table}.created_at", "#{jobs_table}.finished_at"))
|
|
89
|
-
.transform_keys(&:to_i)
|
|
90
|
-
|
|
91
|
-
buckets.map do |bucket|
|
|
92
|
-
value = averages[bucket]
|
|
93
|
-
Point.new(at: Time.at(bucket).utc, seconds: value&.to_f)
|
|
94
|
-
end
|
|
95
|
-
end
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
def total_succeeded = throughput.sum(&:succeeded)
|
|
99
|
-
def total_failed = throughput.sum(&:failed)
|
|
100
|
-
|
|
101
|
-
def empty? = throughput.all? { |point| point.total.zero? }
|
|
102
|
-
|
|
103
|
-
# Finished-job retention shorter than the window means the oldest buckets
|
|
104
|
-
# are missing rows that really did happen. The chart says so rather than
|
|
105
|
-
# showing a decline that is an artefact of purging.
|
|
106
|
-
def truncated_by_retention?
|
|
107
|
-
retention.present? && retention < window.duration
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
def retention
|
|
111
|
-
return nil unless SolidQueue.respond_to?(:clear_finished_jobs_after)
|
|
112
|
-
return nil unless SolidQueue.preserve_finished_jobs?
|
|
113
|
-
|
|
114
|
-
SolidQueue.clear_finished_jobs_after
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
# One query for every queue's hourly finished count, fanned out in Ruby.
|
|
118
|
-
# Used by the queue cards and the Overview mini-table.
|
|
119
|
-
def self.queue_sparklines(now: Time.current, hours: SPARKLINE_HOURS)
|
|
120
|
-
Flightdeck::Cache.fetch("sparklines", hours, align_to(now.to_i, SPARKLINE_BUCKET),
|
|
121
|
-
expires_in: Flightdeck.config.chart_cache_ttl) do
|
|
122
|
-
last = align_to(now.to_i, SPARKLINE_BUCKET)
|
|
123
|
-
slots = Array.new(hours) { |i| last - ((hours - 1 - i) * SPARKLINE_BUCKET) }
|
|
124
|
-
|
|
125
|
-
counts = SolidQueue::Job
|
|
126
|
-
.where(finished_at: Time.at(slots.first).utc..)
|
|
127
|
-
.group(:queue_name)
|
|
128
|
-
.group(TimeBucket.sql("#{SolidQueue::Job.table_name}.finished_at", SPARKLINE_BUCKET))
|
|
129
|
-
.count
|
|
130
|
-
|
|
131
|
-
fanned = Hash.new { |hash, key| hash[key] = Array.new(hours, 0) }
|
|
132
|
-
counts.each do |(queue_name, bucket), count|
|
|
133
|
-
index = slots.index(bucket.to_i)
|
|
134
|
-
next unless index
|
|
135
|
-
|
|
136
|
-
fanned[queue_name][index] = count
|
|
137
|
-
end
|
|
138
|
-
fanned.default = nil
|
|
139
|
-
fanned
|
|
140
|
-
end
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
def self.align_to(epoch, seconds) = (epoch / seconds) * seconds
|
|
144
|
-
|
|
145
|
-
private
|
|
146
|
-
def jobs_table = SolidQueue::Job.table_name
|
|
147
|
-
|
|
148
|
-
def align(epoch) = self.class.align_to(epoch, window.bucket_seconds)
|
|
149
|
-
|
|
150
|
-
def grouped_count(relation, column, table: SolidQueue::Job.table_name)
|
|
151
|
-
relation
|
|
152
|
-
.group(TimeBucket.sql("#{table}.#{column}", window.bucket_seconds))
|
|
153
|
-
.count
|
|
154
|
-
.transform_keys(&:to_i)
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
# Keyed on the aligned bucket so the cache turns over with the buckets
|
|
158
|
-
# themselves rather than at an arbitrary moment.
|
|
159
|
-
def cached(name, &block)
|
|
160
|
-
Flightdeck::Cache.fetch("series", name, window.key, buckets.last,
|
|
161
|
-
expires_in: Flightdeck.config.chart_cache_ttl, &block)
|
|
162
|
-
end
|
|
163
|
-
end
|
|
164
|
-
end
|
|
165
|
-
end
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Flightdeck
|
|
4
|
-
module Metrics
|
|
5
|
-
# Tiny area+line for the queue cards and the Overview mini-table.
|
|
6
|
-
class Sparkline
|
|
7
|
-
WIDTH = 100
|
|
8
|
-
HEIGHT = 30
|
|
9
|
-
|
|
10
|
-
attr_reader :values
|
|
11
|
-
|
|
12
|
-
def initialize(values)
|
|
13
|
-
@values = Array(values).map(&:to_i)
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
def width = WIDTH
|
|
17
|
-
def height = HEIGHT
|
|
18
|
-
def any? = values.any? && values.max.positive?
|
|
19
|
-
|
|
20
|
-
def coordinates
|
|
21
|
-
@coordinates ||= begin
|
|
22
|
-
max = [ values.max.to_i, 1 ].max
|
|
23
|
-
step = values.size > 1 ? WIDTH.to_f / (values.size - 1) : WIDTH.to_f
|
|
24
|
-
|
|
25
|
-
values.each_with_index.map do |value, index|
|
|
26
|
-
{ x: (index * step).round(2),
|
|
27
|
-
y: (HEIGHT - 2 - ((value.to_f / max) * (HEIGHT - 4))).round(2) }
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
def line_path
|
|
33
|
-
coordinates.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def area_path
|
|
37
|
-
return "" if coordinates.empty?
|
|
38
|
-
|
|
39
|
-
"#{line_path} L #{coordinates.last[:x]} #{HEIGHT} L #{coordinates.first[:x]} #{HEIGHT} Z"
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def last_point = coordinates.last
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
end
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Flightdeck
|
|
4
|
-
module Metrics
|
|
5
|
-
# The only adapter-specific SQL in Flightdeck.
|
|
6
|
-
#
|
|
7
|
-
# Charts bucket timestamps into fixed-width windows. Doing that portably
|
|
8
|
-
# means converting a timestamp to a UTC epoch integer and dividing — never
|
|
9
|
-
# date_trunc, which does not exist everywhere and drags the server's
|
|
10
|
-
# timezone into the answer. Everything downstream is an integer number of
|
|
11
|
-
# seconds since the epoch, in UTC; the display timezone is applied at render
|
|
12
|
-
# time and nowhere else.
|
|
13
|
-
module TimeBucket
|
|
14
|
-
class UnsupportedAdapter < StandardError; end
|
|
15
|
-
|
|
16
|
-
# Identifiers are built by Flightdeck, never by a request, but they are
|
|
17
|
-
# still validated before being interpolated so that can never change by
|
|
18
|
-
# accident.
|
|
19
|
-
IDENTIFIER = /\A[a-z_][a-z0-9_]*(\.[a-z_][a-z0-9_]*)?\z/
|
|
20
|
-
|
|
21
|
-
class << self
|
|
22
|
-
def adapter_name
|
|
23
|
-
SolidQueue::Record.connection.adapter_name.to_s
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
# Normalises the adapter into the three SQL dialects we support.
|
|
27
|
-
def family(name = adapter_name)
|
|
28
|
-
case name.to_s.downcase
|
|
29
|
-
when /postgres/ then :postgresql
|
|
30
|
-
when /mysql|trilogy/ then :mysql
|
|
31
|
-
when /sqlite/ then :sqlite
|
|
32
|
-
else
|
|
33
|
-
raise UnsupportedAdapter,
|
|
34
|
-
"Flightdeck cannot bucket time on the #{name} adapter. " \
|
|
35
|
-
"Supported adapters: PostgreSQL, MySQL/Trilogy, SQLite."
|
|
36
|
-
end
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
# Seconds since the UTC epoch, as a number the database can do
|
|
40
|
-
# arithmetic on.
|
|
41
|
-
def epoch(column, family: self.family)
|
|
42
|
-
col = identifier!(column)
|
|
43
|
-
|
|
44
|
-
case family
|
|
45
|
-
when :postgresql then "EXTRACT(EPOCH FROM #{col})"
|
|
46
|
-
when :mysql then "UNIX_TIMESTAMP(#{col})"
|
|
47
|
-
when :sqlite then "CAST(strftime('%s', #{col}) AS INTEGER)"
|
|
48
|
-
else raise UnsupportedAdapter, "unknown family #{family.inspect}"
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
# The epoch second at which `column`'s bucket starts.
|
|
53
|
-
#
|
|
54
|
-
# SQLite gets integer division rather than FLOOR() on purpose: floor() is
|
|
55
|
-
# only available when SQLite is compiled with math functions, and
|
|
56
|
-
# integer division is exact for the post-1970 timestamps we deal with.
|
|
57
|
-
def bucket(column, seconds, family: self.family)
|
|
58
|
-
seconds = interval!(seconds)
|
|
59
|
-
col = identifier!(column)
|
|
60
|
-
|
|
61
|
-
case family
|
|
62
|
-
when :postgresql then "FLOOR(EXTRACT(EPOCH FROM #{col}) / #{seconds}) * #{seconds}"
|
|
63
|
-
when :mysql then "FLOOR(UNIX_TIMESTAMP(#{col}) / #{seconds}) * #{seconds}"
|
|
64
|
-
when :sqlite then "(CAST(strftime('%s', #{col}) AS INTEGER) / #{seconds}) * #{seconds}"
|
|
65
|
-
else raise UnsupportedAdapter, "unknown family #{family.inspect}"
|
|
66
|
-
end
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
def sql(column, seconds) = Arel.sql(bucket(column, seconds))
|
|
70
|
-
|
|
71
|
-
# Seconds between two timestamp columns, for averaging in SQL.
|
|
72
|
-
def elapsed(from_column, to_column, family: self.family)
|
|
73
|
-
"(#{epoch(to_column, family: family)} - #{epoch(from_column, family: family)})"
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
def elapsed_sql(from_column, to_column) = Arel.sql(elapsed(from_column, to_column))
|
|
77
|
-
|
|
78
|
-
private
|
|
79
|
-
def identifier!(column)
|
|
80
|
-
value = column.to_s
|
|
81
|
-
return value if IDENTIFIER.match?(value)
|
|
82
|
-
|
|
83
|
-
raise ArgumentError, "#{column.inspect} is not a plain column identifier"
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
def interval!(seconds)
|
|
87
|
-
value = Integer(seconds)
|
|
88
|
-
return value if value.positive?
|
|
89
|
-
|
|
90
|
-
raise ArgumentError, "bucket width must be a positive number of seconds"
|
|
91
|
-
end
|
|
92
|
-
end
|
|
93
|
-
end
|
|
94
|
-
end
|
|
95
|
-
end
|
|
@@ -1,210 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Flightdeck
|
|
4
|
-
# Everything the Overview page shows, assembled from the same query objects
|
|
5
|
-
# the rest of the dashboard uses.
|
|
6
|
-
class Overview
|
|
7
|
-
WINDOW = 24.hours
|
|
8
|
-
RECENT_FAILURES = 5
|
|
9
|
-
TOP_QUEUES = 6
|
|
10
|
-
|
|
11
|
-
# `value` is always display-ready text — a delimited count, a duration, or
|
|
12
|
-
# "—" — so the view prints it without asking what kind of number it was.
|
|
13
|
-
# trend is :up / :down / :flat and says which way the number moved, while
|
|
14
|
-
# `good` says whether that movement is a good thing — more throughput is
|
|
15
|
-
# good, more failures are not.
|
|
16
|
-
Tile = Struct.new(:label, :value, :unit, :detail, :trend, :good, keyword_init: true)
|
|
17
|
-
|
|
18
|
-
attr_reader :now
|
|
19
|
-
|
|
20
|
-
def initialize(now: Time.current)
|
|
21
|
-
@now = now
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def tiles
|
|
25
|
-
[ processed_tile, failed_tile, ready_tile, scheduled_tile, in_progress_tile, oldest_ready_tile ]
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def series(window: Metrics::Series::DEFAULT_WINDOW)
|
|
29
|
-
@series ||= {}
|
|
30
|
-
@series[window.to_s] ||= Metrics::Series.new(window: window, now: now)
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def queues
|
|
34
|
-
@queues ||= QueueStats.new.rows.sort_by { |row| -row.depth }.first(TOP_QUEUES)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def max_queue_depth
|
|
38
|
-
@max_queue_depth ||= [ queues.map(&:depth).max.to_i, 1 ].max
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
def sparklines
|
|
42
|
-
@sparklines ||= Metrics::Series.queue_sparklines(now: now)
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
def sparkline_for(queue_name)
|
|
46
|
-
Metrics::Sparkline.new(sparklines[queue_name] || [])
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
def recent_failures
|
|
50
|
-
@recent_failures ||= JobsQuery.new(state: :failed, limit: RECENT_FAILURES).rows
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def failed_count
|
|
54
|
-
@failed_count ||= JobsQuery.new(state: :failed).count
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def registry
|
|
58
|
-
@registry ||= ProcessRegistry.new
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# --- counts ---------------------------------------------------------------
|
|
62
|
-
|
|
63
|
-
def processed_24h
|
|
64
|
-
@processed_24h ||= counted("processed", WINDOW) { SolidQueue::Job.where(finished_at: (now - WINDOW)..now) }
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
def processed_prior_24h
|
|
68
|
-
@processed_prior_24h ||= counted("processed_prior", WINDOW) do
|
|
69
|
-
SolidQueue::Job.where(finished_at: (now - (WINDOW * 2))...(now - WINDOW))
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
def failed_24h
|
|
74
|
-
@failed_24h ||= counted("failed", WINDOW) { SolidQueue::FailedExecution.where(created_at: (now - WINDOW)..now) }
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
def ready_now = @ready_now ||= counted("ready") { SolidQueue::ReadyExecution.all }
|
|
78
|
-
def scheduled = @scheduled ||= counted("scheduled") { SolidQueue::ScheduledExecution.all }
|
|
79
|
-
def in_progress = @in_progress ||= counted("in_progress") { SolidQueue::ClaimedExecution.all }
|
|
80
|
-
|
|
81
|
-
def next_scheduled_at
|
|
82
|
-
@next_scheduled_at ||= Cache.fetch("overview", "next_scheduled", expires_in: count_ttl) do
|
|
83
|
-
SolidQueue::ScheduledExecution.minimum(:scheduled_at)
|
|
84
|
-
end
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
def oldest_ready_at
|
|
88
|
-
@oldest_ready_at ||= Cache.fetch("overview", "oldest_ready", expires_in: count_ttl) do
|
|
89
|
-
SolidQueue::ReadyExecution.minimum(:created_at)
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
def oldest_ready_age
|
|
94
|
-
oldest_ready_at && now - oldest_ready_at
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
# Total worker capacity, summed from each worker's registered thread pool
|
|
98
|
-
# size. Returns nil when no worker reported one, so the tile can omit the
|
|
99
|
-
# denominator rather than divide by a number it invented.
|
|
100
|
-
#
|
|
101
|
-
# A worker that has stopped sending heartbeats is not capacity, whatever its
|
|
102
|
-
# metadata still says, so dead workers are excluded — otherwise utilization
|
|
103
|
-
# reads low precisely when the fleet is in trouble.
|
|
104
|
-
def slots
|
|
105
|
-
@slots ||= begin
|
|
106
|
-
sizes = registry.nodes.filter_map do |node|
|
|
107
|
-
next unless node.claims_executions? && !node.dead?
|
|
108
|
-
|
|
109
|
-
Integer(node.metadata.symbolize_keys[:thread_pool_size], exception: false)
|
|
110
|
-
end
|
|
111
|
-
sizes.any? ? sizes.sum : nil
|
|
112
|
-
end
|
|
113
|
-
end
|
|
114
|
-
|
|
115
|
-
def utilization
|
|
116
|
-
return nil if slots.nil? || slots.zero?
|
|
117
|
-
|
|
118
|
-
(in_progress.to_f / slots * 100).round
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
def failure_rate
|
|
122
|
-
total = processed_24h + failed_24h
|
|
123
|
-
return nil if total.zero?
|
|
124
|
-
|
|
125
|
-
(failed_24h.to_f / total * 100).round(1)
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
def processed_delta
|
|
129
|
-
return nil if processed_prior_24h.zero?
|
|
130
|
-
|
|
131
|
-
((processed_24h - processed_prior_24h).to_f / processed_prior_24h * 100).round(1)
|
|
132
|
-
end
|
|
133
|
-
|
|
134
|
-
private
|
|
135
|
-
def count_ttl = Flightdeck.config.poll_interval
|
|
136
|
-
|
|
137
|
-
def counted(name, window = nil, &relation)
|
|
138
|
-
Cache.fetch("overview", name, window&.to_i, expires_in: count_ttl) do
|
|
139
|
-
relation.call.limit(Flightdeck.config.count_cap).count
|
|
140
|
-
end
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
def processed_tile
|
|
144
|
-
delta = processed_delta
|
|
145
|
-
|
|
146
|
-
Tile.new(
|
|
147
|
-
label: "Processed · 24h",
|
|
148
|
-
value: number(processed_24h),
|
|
149
|
-
detail: delta.nil? ? "no prior window to compare" : "#{delta.abs}% vs prior 24h",
|
|
150
|
-
trend: trend_for(delta),
|
|
151
|
-
good: delta.nil? || delta >= 0
|
|
152
|
-
)
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
def failed_tile
|
|
156
|
-
rate = failure_rate
|
|
157
|
-
|
|
158
|
-
Tile.new(
|
|
159
|
-
label: "Failed · 24h",
|
|
160
|
-
value: number(failed_24h),
|
|
161
|
-
detail: rate.nil? ? "nothing finished yet" : "#{rate}% failure rate",
|
|
162
|
-
trend: failed_24h.positive? ? :up : :flat,
|
|
163
|
-
good: failed_24h.zero?
|
|
164
|
-
)
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
def ready_tile
|
|
168
|
-
Tile.new(label: "Ready now", value: number(ready_now), detail: "waiting to be claimed", trend: :flat, good: true)
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
def scheduled_tile
|
|
172
|
-
detail =
|
|
173
|
-
if next_scheduled_at.nil? then "nothing scheduled"
|
|
174
|
-
elsif next_scheduled_at <= now then "due now"
|
|
175
|
-
else "next due in #{Duration.humanize(next_scheduled_at - now)}"
|
|
176
|
-
end
|
|
177
|
-
|
|
178
|
-
Tile.new(label: "Scheduled", value: number(scheduled), detail: detail, trend: :flat, good: true)
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def in_progress_tile
|
|
182
|
-
Tile.new(
|
|
183
|
-
label: "In progress",
|
|
184
|
-
value: number(in_progress),
|
|
185
|
-
unit: slots ? "/ #{number(slots)} slots" : nil,
|
|
186
|
-
detail: utilization ? "#{utilization}% utilization" : "no worker capacity reported",
|
|
187
|
-
trend: :flat,
|
|
188
|
-
good: true
|
|
189
|
-
)
|
|
190
|
-
end
|
|
191
|
-
|
|
192
|
-
def oldest_ready_tile
|
|
193
|
-
Tile.new(
|
|
194
|
-
label: "Oldest ready",
|
|
195
|
-
value: oldest_ready_age ? Duration.humanize(oldest_ready_age) : "—",
|
|
196
|
-
detail: oldest_ready_age ? "longest a job has waited" : "nothing waiting",
|
|
197
|
-
trend: :flat,
|
|
198
|
-
good: true
|
|
199
|
-
)
|
|
200
|
-
end
|
|
201
|
-
|
|
202
|
-
def number(value) = ActiveSupport::NumberHelper.number_to_delimited(value)
|
|
203
|
-
|
|
204
|
-
def trend_for(delta)
|
|
205
|
-
return :flat if delta.nil? || delta.abs < 0.05
|
|
206
|
-
|
|
207
|
-
delta.positive? ? :up : :down
|
|
208
|
-
end
|
|
209
|
-
end
|
|
210
|
-
end
|