flightdeck 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/MIT-LICENSE +21 -0
- data/README.md +190 -0
- data/app/assets/flightdeck/flightdeck-00403457c5d9.css +3 -0
- data/app/assets/flightdeck/flightdeck-c02e38040ddb.js +414 -0
- data/app/assets/flightdeck/manifest.json +16 -0
- data/app/controllers/concerns/flightdeck/toasts.rb +25 -0
- data/app/controllers/flightdeck/application_controller.rb +96 -0
- data/app/controllers/flightdeck/assets_controller.rb +39 -0
- data/app/controllers/flightdeck/jobs/actions_controller.rb +167 -0
- data/app/controllers/flightdeck/jobs/discards_controller.rb +44 -0
- data/app/controllers/flightdeck/jobs/retries_controller.rb +23 -0
- data/app/controllers/flightdeck/jobs_controller.rb +39 -0
- data/app/controllers/flightdeck/overview_controller.rb +11 -0
- data/app/controllers/flightdeck/processes_controller.rb +49 -0
- data/app/controllers/flightdeck/queues_controller.rb +51 -0
- data/app/controllers/flightdeck/recurring_tasks_controller.rb +46 -0
- data/app/helpers/flightdeck/application_helper.rb +68 -0
- data/app/helpers/flightdeck/infrastructure_helper.rb +24 -0
- data/app/helpers/flightdeck/jobs_helper.rb +136 -0
- data/app/helpers/flightdeck/metrics_helper.rb +49 -0
- data/app/models/flightdeck/arguments_preview.rb +95 -0
- data/app/models/flightdeck/bulk_action.rb +83 -0
- data/app/models/flightdeck/cache.rb +51 -0
- data/app/models/flightdeck/cron_schedule.rb +102 -0
- data/app/models/flightdeck/error_summary.rb +64 -0
- data/app/models/flightdeck/grouped_failures.rb +24 -0
- data/app/models/flightdeck/job_detail.rb +163 -0
- data/app/models/flightdeck/job_row.rb +93 -0
- data/app/models/flightdeck/jobs_query.rb +241 -0
- data/app/models/flightdeck/metrics/bar_chart.rb +49 -0
- data/app/models/flightdeck/metrics/chart.rb +111 -0
- data/app/models/flightdeck/metrics/line_chart.rb +67 -0
- data/app/models/flightdeck/metrics/series.rb +165 -0
- data/app/models/flightdeck/metrics/sparkline.rb +45 -0
- data/app/models/flightdeck/metrics/time_bucket.rb +95 -0
- data/app/models/flightdeck/overview.rb +217 -0
- data/app/models/flightdeck/process_registry.rb +108 -0
- data/app/models/flightdeck/queue_stats.rb +88 -0
- data/app/models/flightdeck/recurring_catalog.rb +116 -0
- data/app/views/flightdeck/jobs/_failed_row.html.erb +33 -0
- data/app/views/flightdeck/jobs/_filter_bar.html.erb +29 -0
- data/app/views/flightdeck/jobs/_list.html.erb +89 -0
- data/app/views/flightdeck/jobs/_pager.html.erb +22 -0
- data/app/views/flightdeck/jobs/_row.html.erb +23 -0
- data/app/views/flightdeck/jobs/_state_tabs.html.erb +10 -0
- data/app/views/flightdeck/jobs/actions/create.turbo_stream.erb +18 -0
- data/app/views/flightdeck/jobs/index.html.erb +13 -0
- data/app/views/flightdeck/jobs/show.html.erb +100 -0
- data/app/views/flightdeck/metrics/_bar_chart.html.erb +34 -0
- data/app/views/flightdeck/metrics/_line_chart.html.erb +38 -0
- data/app/views/flightdeck/metrics/_range_control.html.erb +7 -0
- data/app/views/flightdeck/metrics/_sparkline.html.erb +9 -0
- data/app/views/flightdeck/overview/_completion.html.erb +19 -0
- data/app/views/flightdeck/overview/_failures.html.erb +28 -0
- data/app/views/flightdeck/overview/_fleet.html.erb +21 -0
- data/app/views/flightdeck/overview/_queues.html.erb +41 -0
- data/app/views/flightdeck/overview/_throughput.html.erb +23 -0
- data/app/views/flightdeck/overview/_tiles.html.erb +14 -0
- data/app/views/flightdeck/overview/index.html.erb +54 -0
- data/app/views/flightdeck/processes/_fleet.html.erb +54 -0
- data/app/views/flightdeck/processes/_row.html.erb +28 -0
- data/app/views/flightdeck/processes/index.html.erb +9 -0
- data/app/views/flightdeck/processes/prune.turbo_stream.erb +15 -0
- data/app/views/flightdeck/queues/_cards.html.erb +46 -0
- data/app/views/flightdeck/queues/index.html.erb +9 -0
- data/app/views/flightdeck/queues/update.turbo_stream.erb +15 -0
- data/app/views/flightdeck/recurring_tasks/_table.html.erb +53 -0
- data/app/views/flightdeck/recurring_tasks/index.html.erb +9 -0
- data/app/views/flightdeck/recurring_tasks/run.turbo_stream.erb +15 -0
- data/app/views/flightdeck/shared/_toast.html.erb +7 -0
- data/app/views/layouts/flightdeck/application.html.erb +110 -0
- data/config/routes.rb +40 -0
- data/lib/flightdeck/assets.rb +78 -0
- data/lib/flightdeck/configuration.rb +80 -0
- data/lib/flightdeck/engine.rb +31 -0
- data/lib/flightdeck/version.rb +5 -0
- data/lib/flightdeck.rb +31 -0
- metadata +150 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flightdeck
|
|
4
|
+
module Metrics
|
|
5
|
+
# Average completion time: area under a line, with gaps where no jobs
|
|
6
|
+
# finished at all.
|
|
7
|
+
class LineChart < Chart
|
|
8
|
+
DOT_RADIUS = 3
|
|
9
|
+
|
|
10
|
+
def coordinates
|
|
11
|
+
@coordinates ||= points.each_with_index.map do |point, index|
|
|
12
|
+
next nil if point.seconds.nil?
|
|
13
|
+
|
|
14
|
+
{ x: round(plot_left + (slot_width * index) + (slot_width / 2.0)),
|
|
15
|
+
y: round(y_for(point.seconds)),
|
|
16
|
+
seconds: point.seconds,
|
|
17
|
+
title: point_title(point) }
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Contiguous runs of buckets that had data. Drawing them separately keeps
|
|
22
|
+
# a gap in the data looking like a gap, instead of a straight line across
|
|
23
|
+
# a period we know nothing about.
|
|
24
|
+
def segments
|
|
25
|
+
coordinates.chunk_while { |a, b| !a.nil? && !b.nil? }.filter_map do |chunk|
|
|
26
|
+
present = chunk.compact
|
|
27
|
+
present.size.positive? ? present : nil
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def line_path(segment)
|
|
32
|
+
return "" if segment.empty?
|
|
33
|
+
|
|
34
|
+
segment.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def area_path(segment)
|
|
38
|
+
return "" if segment.empty?
|
|
39
|
+
|
|
40
|
+
"#{line_path(segment)} L #{segment.last[:x]} #{plot_bottom} L #{segment.first[:x]} #{plot_bottom} Z"
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def markers = coordinates.compact
|
|
44
|
+
|
|
45
|
+
def last_marker = markers.last
|
|
46
|
+
|
|
47
|
+
def any_data? = markers.any?
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
def raw_max = points.filter_map(&:seconds).max.to_f
|
|
51
|
+
|
|
52
|
+
def format_value(value)
|
|
53
|
+
case value
|
|
54
|
+
when 0 then "0s"
|
|
55
|
+
when 0...1 then "#{(value * 1000).round}ms"
|
|
56
|
+
when 1...60 then "#{value.round(1).to_s.sub(/\.0\z/, "")}s"
|
|
57
|
+
when 60...3600 then "#{(value / 60).round}m"
|
|
58
|
+
else "#{(value / 3600).round(1)}h"
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def point_title(point)
|
|
63
|
+
"#{full_time(point.at)} · avg #{format_value(point.seconds)} to completion"
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flightdeck
|
|
4
|
+
module Metrics
|
|
5
|
+
# Time series for the Overview charts, computed live from the Solid Queue
|
|
6
|
+
# tables — no rollup tables, no background aggregation.
|
|
7
|
+
#
|
|
8
|
+
# Every series is gap-filled in Ruby so a chart always has a complete axis:
|
|
9
|
+
# a bucket with no rows is a real zero (or a real "no data"), not a missing
|
|
10
|
+
# point that would make the chart lie about its own shape.
|
|
11
|
+
class Series
|
|
12
|
+
Window = Struct.new(:key, :label, :duration, :bucket_seconds, :subtitle, :tick_every,
|
|
13
|
+
:tick_format, keyword_init: true) do
|
|
14
|
+
def bucket_count = (duration / bucket_seconds).to_i
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
WINDOWS = {
|
|
18
|
+
"1h" => Window.new(key: "1h", label: "1H", duration: 1.hour, bucket_seconds: 5.minutes.to_i,
|
|
19
|
+
subtitle: "per 5 minutes", tick_every: 3, tick_format: "%H:%M"),
|
|
20
|
+
"24h" => Window.new(key: "24h", label: "24H", duration: 24.hours, bucket_seconds: 1.hour.to_i,
|
|
21
|
+
subtitle: "per hour", tick_every: 4, tick_format: "%H:%M"),
|
|
22
|
+
"7d" => Window.new(key: "7d", label: "7D", duration: 7.days, bucket_seconds: 6.hours.to_i,
|
|
23
|
+
subtitle: "per 6 hours", tick_every: 4, tick_format: "%-m/%-d")
|
|
24
|
+
}.freeze
|
|
25
|
+
|
|
26
|
+
DEFAULT_WINDOW = "24h"
|
|
27
|
+
SPARKLINE_HOURS = 16
|
|
28
|
+
SPARKLINE_BUCKET = 1.hour.to_i
|
|
29
|
+
|
|
30
|
+
Point = Struct.new(:at, :succeeded, :failed, :seconds, keyword_init: true) do
|
|
31
|
+
def total = succeeded.to_i + failed.to_i
|
|
32
|
+
def blank? = seconds.nil?
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
attr_reader :window, :now
|
|
36
|
+
|
|
37
|
+
def self.window_for(key)
|
|
38
|
+
WINDOWS.fetch(key.to_s, WINDOWS.fetch(DEFAULT_WINDOW))
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def initialize(window: DEFAULT_WINDOW, now: Time.current)
|
|
42
|
+
@window = window.is_a?(Window) ? window : self.class.window_for(window)
|
|
43
|
+
@now = now.utc
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Bucket start times, aligned to absolute epoch multiples so that the same
|
|
47
|
+
# bucket has the same boundaries in every process and every request.
|
|
48
|
+
def buckets
|
|
49
|
+
@buckets ||= begin
|
|
50
|
+
last = align(@now.to_i)
|
|
51
|
+
count = window.bucket_count
|
|
52
|
+
Array.new(count) { |i| last - ((count - 1 - i) * window.bucket_seconds) }
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def starts_at = Time.at(buckets.first).utc
|
|
57
|
+
def ends_at = Time.at(buckets.last + window.bucket_seconds).utc
|
|
58
|
+
|
|
59
|
+
# Jobs finished (succeeded) and executions failed, per bucket.
|
|
60
|
+
#
|
|
61
|
+
# A job that failed has no finished_at — Solid Queue leaves it null and
|
|
62
|
+
# records a failed_executions row — so the two series never double count.
|
|
63
|
+
def throughput
|
|
64
|
+
cached(:throughput) do
|
|
65
|
+
succeeded = grouped_count(SolidQueue::Job.where(finished_at: starts_at..), :finished_at)
|
|
66
|
+
failed = grouped_count(SolidQueue::FailedExecution.where(created_at: starts_at..), :created_at,
|
|
67
|
+
table: SolidQueue::FailedExecution.table_name)
|
|
68
|
+
|
|
69
|
+
buckets.map do |bucket|
|
|
70
|
+
Point.new(at: Time.at(bucket).utc,
|
|
71
|
+
succeeded: succeeded.fetch(bucket, 0),
|
|
72
|
+
failed: failed.fetch(bucket, 0))
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Average wall-clock time from enqueue to finish, per bucket.
|
|
78
|
+
#
|
|
79
|
+
# This is time *to completion*, not time to start: Solid Queue deletes the
|
|
80
|
+
# claimed_executions row when a job finishes, so the moment a job actually
|
|
81
|
+
# started is not recoverable after the fact and no honest historical
|
|
82
|
+
# time-to-start series can be built from these tables.
|
|
83
|
+
def completion_time
|
|
84
|
+
cached(:completion_time) do
|
|
85
|
+
averages = SolidQueue::Job
|
|
86
|
+
.where(finished_at: starts_at..)
|
|
87
|
+
.group(TimeBucket.sql("#{jobs_table}.finished_at", window.bucket_seconds))
|
|
88
|
+
.average(TimeBucket.elapsed_sql("#{jobs_table}.created_at", "#{jobs_table}.finished_at"))
|
|
89
|
+
.transform_keys(&:to_i)
|
|
90
|
+
|
|
91
|
+
buckets.map do |bucket|
|
|
92
|
+
value = averages[bucket]
|
|
93
|
+
Point.new(at: Time.at(bucket).utc, seconds: value&.to_f)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def total_succeeded = throughput.sum(&:succeeded)
|
|
99
|
+
def total_failed = throughput.sum(&:failed)
|
|
100
|
+
|
|
101
|
+
def empty? = throughput.all? { |point| point.total.zero? }
|
|
102
|
+
|
|
103
|
+
# Finished-job retention shorter than the window means the oldest buckets
|
|
104
|
+
# are missing rows that really did happen. The chart says so rather than
|
|
105
|
+
# showing a decline that is an artefact of purging.
|
|
106
|
+
def truncated_by_retention?
|
|
107
|
+
retention.present? && retention < window.duration
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def retention
|
|
111
|
+
return nil unless SolidQueue.respond_to?(:clear_finished_jobs_after)
|
|
112
|
+
return nil unless SolidQueue.preserve_finished_jobs?
|
|
113
|
+
|
|
114
|
+
SolidQueue.clear_finished_jobs_after
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# One query for every queue's hourly finished count, fanned out in Ruby.
|
|
118
|
+
# Used by the queue cards and the Overview mini-table.
|
|
119
|
+
def self.queue_sparklines(now: Time.current, hours: SPARKLINE_HOURS)
|
|
120
|
+
Flightdeck::Cache.fetch("sparklines", hours, align_to(now.to_i, SPARKLINE_BUCKET),
|
|
121
|
+
expires_in: Flightdeck.config.chart_cache_ttl) do
|
|
122
|
+
last = align_to(now.to_i, SPARKLINE_BUCKET)
|
|
123
|
+
slots = Array.new(hours) { |i| last - ((hours - 1 - i) * SPARKLINE_BUCKET) }
|
|
124
|
+
|
|
125
|
+
counts = SolidQueue::Job
|
|
126
|
+
.where(finished_at: Time.at(slots.first).utc..)
|
|
127
|
+
.group(:queue_name)
|
|
128
|
+
.group(TimeBucket.sql("#{SolidQueue::Job.table_name}.finished_at", SPARKLINE_BUCKET))
|
|
129
|
+
.count
|
|
130
|
+
|
|
131
|
+
fanned = Hash.new { |hash, key| hash[key] = Array.new(hours, 0) }
|
|
132
|
+
counts.each do |(queue_name, bucket), count|
|
|
133
|
+
index = slots.index(bucket.to_i)
|
|
134
|
+
next unless index
|
|
135
|
+
|
|
136
|
+
fanned[queue_name][index] = count
|
|
137
|
+
end
|
|
138
|
+
fanned.default = nil
|
|
139
|
+
fanned
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def self.align_to(epoch, seconds) = (epoch / seconds) * seconds
|
|
144
|
+
|
|
145
|
+
private
|
|
146
|
+
def jobs_table = SolidQueue::Job.table_name
|
|
147
|
+
|
|
148
|
+
def align(epoch) = self.class.align_to(epoch, window.bucket_seconds)
|
|
149
|
+
|
|
150
|
+
def grouped_count(relation, column, table: SolidQueue::Job.table_name)
|
|
151
|
+
relation
|
|
152
|
+
.group(TimeBucket.sql("#{table}.#{column}", window.bucket_seconds))
|
|
153
|
+
.count
|
|
154
|
+
.transform_keys(&:to_i)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Keyed on the aligned bucket so the cache turns over with the buckets
|
|
158
|
+
# themselves rather than at an arbitrary moment.
|
|
159
|
+
def cached(name, &block)
|
|
160
|
+
Flightdeck::Cache.fetch("series", name, window.key, buckets.last,
|
|
161
|
+
expires_in: Flightdeck.config.chart_cache_ttl, &block)
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flightdeck
|
|
4
|
+
module Metrics
|
|
5
|
+
# Tiny area+line for the queue cards and the Overview mini-table.
|
|
6
|
+
class Sparkline
|
|
7
|
+
WIDTH = 100
|
|
8
|
+
HEIGHT = 30
|
|
9
|
+
|
|
10
|
+
attr_reader :values
|
|
11
|
+
|
|
12
|
+
def initialize(values)
|
|
13
|
+
@values = Array(values).map(&:to_i)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def width = WIDTH
|
|
17
|
+
def height = HEIGHT
|
|
18
|
+
def any? = values.any? && values.max.positive?
|
|
19
|
+
|
|
20
|
+
def coordinates
|
|
21
|
+
@coordinates ||= begin
|
|
22
|
+
max = [ values.max.to_i, 1 ].max
|
|
23
|
+
step = values.size > 1 ? WIDTH.to_f / (values.size - 1) : WIDTH.to_f
|
|
24
|
+
|
|
25
|
+
values.each_with_index.map do |value, index|
|
|
26
|
+
{ x: (index * step).round(2),
|
|
27
|
+
y: (HEIGHT - 2 - ((value.to_f / max) * (HEIGHT - 4))).round(2) }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def line_path
|
|
33
|
+
coordinates.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def area_path
|
|
37
|
+
return "" if coordinates.empty?
|
|
38
|
+
|
|
39
|
+
"#{line_path} L #{coordinates.last[:x]} #{HEIGHT} L #{coordinates.first[:x]} #{HEIGHT} Z"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def last_point = coordinates.last
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flightdeck
|
|
4
|
+
module Metrics
|
|
5
|
+
# The only adapter-specific SQL in Flightdeck.
|
|
6
|
+
#
|
|
7
|
+
# Charts bucket timestamps into fixed-width windows. Doing that portably
|
|
8
|
+
# means converting a timestamp to a UTC epoch integer and dividing — never
|
|
9
|
+
# date_trunc, which does not exist everywhere and drags the server's
|
|
10
|
+
# timezone into the answer. Everything downstream is an integer number of
|
|
11
|
+
# seconds since the epoch, in UTC; the display timezone is applied at render
|
|
12
|
+
# time and nowhere else.
|
|
13
|
+
module TimeBucket
|
|
14
|
+
class UnsupportedAdapter < StandardError; end
|
|
15
|
+
|
|
16
|
+
# Identifiers are built by Flightdeck, never by a request, but they are
|
|
17
|
+
# still validated before being interpolated so that can never change by
|
|
18
|
+
# accident.
|
|
19
|
+
IDENTIFIER = /\A[a-z_][a-z0-9_]*(\.[a-z_][a-z0-9_]*)?\z/
|
|
20
|
+
|
|
21
|
+
class << self
|
|
22
|
+
def adapter_name
|
|
23
|
+
SolidQueue::Record.connection.adapter_name.to_s
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Normalises the adapter into the three SQL dialects we support.
|
|
27
|
+
def family(name = adapter_name)
|
|
28
|
+
case name.to_s.downcase
|
|
29
|
+
when /postgres/ then :postgresql
|
|
30
|
+
when /mysql|trilogy/ then :mysql
|
|
31
|
+
when /sqlite/ then :sqlite
|
|
32
|
+
else
|
|
33
|
+
raise UnsupportedAdapter,
|
|
34
|
+
"Flightdeck cannot bucket time on the #{name} adapter. " \
|
|
35
|
+
"Supported adapters: PostgreSQL, MySQL/Trilogy, SQLite."
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Seconds since the UTC epoch, as a number the database can do
|
|
40
|
+
# arithmetic on.
|
|
41
|
+
def epoch(column, family: self.family)
|
|
42
|
+
col = identifier!(column)
|
|
43
|
+
|
|
44
|
+
case family
|
|
45
|
+
when :postgresql then "EXTRACT(EPOCH FROM #{col})"
|
|
46
|
+
when :mysql then "UNIX_TIMESTAMP(#{col})"
|
|
47
|
+
when :sqlite then "CAST(strftime('%s', #{col}) AS INTEGER)"
|
|
48
|
+
else raise UnsupportedAdapter, "unknown family #{family.inspect}"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The epoch second at which `column`'s bucket starts.
|
|
53
|
+
#
|
|
54
|
+
# SQLite gets integer division rather than FLOOR() on purpose: floor() is
|
|
55
|
+
# only available when SQLite is compiled with math functions, and
|
|
56
|
+
# integer division is exact for the post-1970 timestamps we deal with.
|
|
57
|
+
def bucket(column, seconds, family: self.family)
|
|
58
|
+
seconds = interval!(seconds)
|
|
59
|
+
col = identifier!(column)
|
|
60
|
+
|
|
61
|
+
case family
|
|
62
|
+
when :postgresql then "FLOOR(EXTRACT(EPOCH FROM #{col}) / #{seconds}) * #{seconds}"
|
|
63
|
+
when :mysql then "FLOOR(UNIX_TIMESTAMP(#{col}) / #{seconds}) * #{seconds}"
|
|
64
|
+
when :sqlite then "(CAST(strftime('%s', #{col}) AS INTEGER) / #{seconds}) * #{seconds}"
|
|
65
|
+
else raise UnsupportedAdapter, "unknown family #{family.inspect}"
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def sql(column, seconds) = Arel.sql(bucket(column, seconds))
|
|
70
|
+
|
|
71
|
+
# Seconds between two timestamp columns, for averaging in SQL.
|
|
72
|
+
def elapsed(from_column, to_column, family: self.family)
|
|
73
|
+
"(#{epoch(to_column, family: family)} - #{epoch(from_column, family: family)})"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def elapsed_sql(from_column, to_column) = Arel.sql(elapsed(from_column, to_column))
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
def identifier!(column)
|
|
80
|
+
value = column.to_s
|
|
81
|
+
return value if IDENTIFIER.match?(value)
|
|
82
|
+
|
|
83
|
+
raise ArgumentError, "#{column.inspect} is not a plain column identifier"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def interval!(seconds)
|
|
87
|
+
value = Integer(seconds)
|
|
88
|
+
return value if value.positive?
|
|
89
|
+
|
|
90
|
+
raise ArgumentError, "bucket width must be a positive number of seconds"
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flightdeck
|
|
4
|
+
# Everything the Overview page shows, assembled from the same query objects
|
|
5
|
+
# the rest of the dashboard uses.
|
|
6
|
+
class Overview
|
|
7
|
+
WINDOW = 24.hours
|
|
8
|
+
RECENT_FAILURES = 5
|
|
9
|
+
TOP_QUEUES = 6
|
|
10
|
+
|
|
11
|
+
# trend is :up / :down / :flat and says which way the number moved, while
|
|
12
|
+
# `good` says whether that movement is a good thing — more throughput is
|
|
13
|
+
# good, more failures are not.
|
|
14
|
+
Tile = Struct.new(:label, :value, :unit, :detail, :trend, :good, keyword_init: true)
|
|
15
|
+
|
|
16
|
+
attr_reader :now
|
|
17
|
+
|
|
18
|
+
def initialize(now: Time.current)
|
|
19
|
+
@now = now
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def tiles
|
|
23
|
+
[ processed_tile, failed_tile, ready_tile, scheduled_tile, in_progress_tile, oldest_ready_tile ]
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def series(window: Metrics::Series::DEFAULT_WINDOW)
|
|
27
|
+
@series ||= {}
|
|
28
|
+
@series[window.to_s] ||= Metrics::Series.new(window: window, now: now)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def queues
|
|
32
|
+
@queues ||= QueueStats.new.rows.sort_by { |row| -row.depth }.first(TOP_QUEUES)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def max_queue_depth
|
|
36
|
+
@max_queue_depth ||= [ queues.map(&:depth).max.to_i, 1 ].max
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def sparklines
|
|
40
|
+
@sparklines ||= Metrics::Series.queue_sparklines(now: now)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def sparkline_for(queue_name)
|
|
44
|
+
Metrics::Sparkline.new(sparklines[queue_name] || [])
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def recent_failures
|
|
48
|
+
@recent_failures ||= JobsQuery.new(state: :failed, limit: RECENT_FAILURES).rows
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def failed_count
|
|
52
|
+
@failed_count ||= JobsQuery.new(state: :failed).count
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def registry
|
|
56
|
+
@registry ||= ProcessRegistry.new
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# --- counts ---------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
def processed_24h
|
|
62
|
+
@processed_24h ||= counted("processed", WINDOW) { SolidQueue::Job.where(finished_at: (now - WINDOW)..now) }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def processed_prior_24h
|
|
66
|
+
@processed_prior_24h ||= counted("processed_prior", WINDOW) do
|
|
67
|
+
SolidQueue::Job.where(finished_at: (now - (WINDOW * 2))...(now - WINDOW))
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def failed_24h
|
|
72
|
+
@failed_24h ||= counted("failed", WINDOW) { SolidQueue::FailedExecution.where(created_at: (now - WINDOW)..now) }
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def ready_now = @ready_now ||= counted("ready") { SolidQueue::ReadyExecution.all }
|
|
76
|
+
def scheduled = @scheduled ||= counted("scheduled") { SolidQueue::ScheduledExecution.all }
|
|
77
|
+
def in_progress = @in_progress ||= counted("in_progress") { SolidQueue::ClaimedExecution.all }
|
|
78
|
+
|
|
79
|
+
def next_scheduled_at
|
|
80
|
+
@next_scheduled_at ||= Cache.fetch("overview", "next_scheduled", expires_in: count_ttl) do
|
|
81
|
+
SolidQueue::ScheduledExecution.minimum(:scheduled_at)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def oldest_ready_at
|
|
86
|
+
@oldest_ready_at ||= Cache.fetch("overview", "oldest_ready", expires_in: count_ttl) do
|
|
87
|
+
SolidQueue::ReadyExecution.minimum(:created_at)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def oldest_ready_age
|
|
92
|
+
oldest_ready_at && now - oldest_ready_at
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Total worker capacity, summed from each worker's registered thread pool
|
|
96
|
+
# size. Returns nil when no worker reported one, so the tile can omit the
|
|
97
|
+
# denominator rather than divide by a number it invented.
|
|
98
|
+
#
|
|
99
|
+
# A worker that has stopped sending heartbeats is not capacity, whatever its
|
|
100
|
+
# metadata still says, so dead workers are excluded — otherwise utilization
|
|
101
|
+
# reads low precisely when the fleet is in trouble.
|
|
102
|
+
def slots
|
|
103
|
+
@slots ||= begin
|
|
104
|
+
sizes = registry.nodes.filter_map do |node|
|
|
105
|
+
next unless node.claims_executions? && !node.dead?
|
|
106
|
+
|
|
107
|
+
Integer(node.metadata.symbolize_keys[:thread_pool_size], exception: false)
|
|
108
|
+
end
|
|
109
|
+
sizes.any? ? sizes.sum : nil
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def utilization
|
|
114
|
+
return nil if slots.nil? || slots.zero?
|
|
115
|
+
|
|
116
|
+
(in_progress.to_f / slots * 100).round
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def failure_rate
|
|
120
|
+
total = processed_24h + failed_24h
|
|
121
|
+
return nil if total.zero?
|
|
122
|
+
|
|
123
|
+
(failed_24h.to_f / total * 100).round(1)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def processed_delta
|
|
127
|
+
return nil if processed_prior_24h.zero?
|
|
128
|
+
|
|
129
|
+
((processed_24h - processed_prior_24h).to_f / processed_prior_24h * 100).round(1)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
private
|
|
133
|
+
def count_ttl = Flightdeck.config.poll_interval
|
|
134
|
+
|
|
135
|
+
def counted(name, window = nil, &relation)
|
|
136
|
+
Cache.fetch("overview", name, window&.to_i, expires_in: count_ttl) do
|
|
137
|
+
relation.call.limit(Flightdeck.config.count_cap).count
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def processed_tile
|
|
142
|
+
delta = processed_delta
|
|
143
|
+
|
|
144
|
+
Tile.new(
|
|
145
|
+
label: "Processed · 24h",
|
|
146
|
+
value: processed_24h,
|
|
147
|
+
detail: delta.nil? ? "no prior window to compare" : "#{delta.abs}% vs prior 24h",
|
|
148
|
+
trend: trend_for(delta),
|
|
149
|
+
good: delta.nil? || delta >= 0
|
|
150
|
+
)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def failed_tile
|
|
154
|
+
rate = failure_rate
|
|
155
|
+
|
|
156
|
+
Tile.new(
|
|
157
|
+
label: "Failed · 24h",
|
|
158
|
+
value: failed_24h,
|
|
159
|
+
detail: rate.nil? ? "nothing finished yet" : "#{rate}% failure rate",
|
|
160
|
+
trend: failed_24h.positive? ? :up : :flat,
|
|
161
|
+
good: failed_24h.zero?
|
|
162
|
+
)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def ready_tile
|
|
166
|
+
Tile.new(label: "Ready now", value: ready_now, detail: "waiting to be claimed", trend: :flat, good: true)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def scheduled_tile
|
|
170
|
+
detail =
|
|
171
|
+
if next_scheduled_at.nil? then "nothing scheduled"
|
|
172
|
+
elsif next_scheduled_at <= now then "due now"
|
|
173
|
+
else "next due in #{humanize_duration(next_scheduled_at - now)}"
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
Tile.new(label: "Scheduled", value: scheduled, detail: detail, trend: :flat, good: true)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def in_progress_tile
|
|
180
|
+
Tile.new(
|
|
181
|
+
label: "In progress",
|
|
182
|
+
value: in_progress,
|
|
183
|
+
unit: slots ? "/ #{ActiveSupport::NumberHelper.number_to_delimited(slots)} slots" : nil,
|
|
184
|
+
detail: utilization ? "#{utilization}% utilization" : "no worker capacity reported",
|
|
185
|
+
trend: :flat,
|
|
186
|
+
good: true
|
|
187
|
+
)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def oldest_ready_tile
|
|
191
|
+
Tile.new(
|
|
192
|
+
label: "Oldest ready",
|
|
193
|
+
value: oldest_ready_age ? humanize_duration(oldest_ready_age) : "—",
|
|
194
|
+
detail: oldest_ready_age ? "longest a job has waited" : "nothing waiting",
|
|
195
|
+
trend: :flat,
|
|
196
|
+
good: true
|
|
197
|
+
)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def trend_for(delta)
|
|
201
|
+
return :flat if delta.nil? || delta.abs < 0.05
|
|
202
|
+
|
|
203
|
+
delta.positive? ? :up : :down
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def humanize_duration(seconds)
|
|
207
|
+
seconds = seconds.to_f.abs
|
|
208
|
+
case seconds
|
|
209
|
+
when 0...1 then "#{(seconds * 1000).round}ms"
|
|
210
|
+
when 1...60 then "#{seconds.round(1).to_s.sub(/\.0\z/, "")}s"
|
|
211
|
+
when 60...3600 then "#{(seconds / 60).round}m"
|
|
212
|
+
when 3600...86_400 then "#{(seconds / 3600).round(1)}h"
|
|
213
|
+
else "#{(seconds / 86_400).round(1)}d"
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flightdeck
|
|
4
|
+
# The Solid Queue fleet: every registered process, grouped supervisor →
|
|
5
|
+
# children, with heartbeat freshness and claimed counts.
|
|
6
|
+
#
|
|
7
|
+
# Two queries total, regardless of fleet size.
|
|
8
|
+
class ProcessRegistry
|
|
9
|
+
# Freshness is measured against Solid Queue's own liveness threshold rather
|
|
10
|
+
# than a number of our own, so what Flightdeck calls dead is exactly what
|
|
11
|
+
# the pruner will collect.
|
|
12
|
+
FRESH_FRACTION = 0.2
|
|
13
|
+
|
|
14
|
+
Node = Struct.new(:process, :claimed_count, :children, keyword_init: true) do
|
|
15
|
+
def id = process.id
|
|
16
|
+
def kind = process.kind
|
|
17
|
+
def name = process.name
|
|
18
|
+
def hostname = process.hostname
|
|
19
|
+
def pid = process.pid
|
|
20
|
+
def last_heartbeat_at = process.last_heartbeat_at
|
|
21
|
+
def supervisor? = kind == "Supervisor"
|
|
22
|
+
def claims_executions? = kind == "Worker"
|
|
23
|
+
|
|
24
|
+
def heartbeat_age
|
|
25
|
+
Time.current - last_heartbeat_at
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def freshness
|
|
29
|
+
threshold = SolidQueue.process_alive_threshold
|
|
30
|
+
age = heartbeat_age
|
|
31
|
+
|
|
32
|
+
if age >= threshold then :dead
|
|
33
|
+
elsif age >= threshold * FRESH_FRACTION then :stale
|
|
34
|
+
else :fresh
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def dead? = freshness == :dead
|
|
39
|
+
def prunable? = dead?
|
|
40
|
+
|
|
41
|
+
def metadata
|
|
42
|
+
process.metadata.presence || {}
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Human summary of what this process was configured to do. Keys differ per
|
|
46
|
+
# process kind, so known ones are formatted and anything else is shown
|
|
47
|
+
# verbatim rather than dropped.
|
|
48
|
+
def config_summary
|
|
49
|
+
parts = []
|
|
50
|
+
data = metadata.symbolize_keys
|
|
51
|
+
|
|
52
|
+
parts << "queues: #{data[:queues]}" if data[:queues].present?
|
|
53
|
+
parts << "#{data[:thread_pool_size]} threads" if data[:thread_pool_size].present?
|
|
54
|
+
parts << "batch #{data[:batch_size]}" if data[:batch_size].present?
|
|
55
|
+
parts << "every #{data[:polling_interval]}s" if data[:polling_interval].present?
|
|
56
|
+
parts << "#{Array(data[:recurring_schedule]).size} recurring tasks" if data[:recurring_schedule].present?
|
|
57
|
+
|
|
58
|
+
known = %i[queues thread_pool_size batch_size polling_interval recurring_schedule
|
|
59
|
+
concurrency_maintenance_interval]
|
|
60
|
+
data.except(*known).each { |key, value| parts << "#{key}: #{value}" }
|
|
61
|
+
|
|
62
|
+
parts.join(" · ")
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.all = new
|
|
67
|
+
|
|
68
|
+
def nodes
|
|
69
|
+
@nodes ||= build_nodes
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Supervisors with their children nested; anything unsupervised (or whose
|
|
73
|
+
# supervisor row has gone) is listed flat so it can never be hidden.
|
|
74
|
+
def tree
|
|
75
|
+
@tree ||= begin
|
|
76
|
+
by_supervisor = nodes.group_by { |node| node.process.supervisor_id }
|
|
77
|
+
known_ids = nodes.map(&:id).to_set
|
|
78
|
+
|
|
79
|
+
roots = nodes.select { |node| node.process.supervisor_id.nil? || !known_ids.include?(node.process.supervisor_id) }
|
|
80
|
+
roots.each { |root| root.children = by_supervisor.fetch(root.id, []) }
|
|
81
|
+
roots
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def flattened
|
|
86
|
+
tree.flat_map { |root| [ root, *root.children ] }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def dead = nodes.select(&:dead?)
|
|
90
|
+
def any_dead? = dead.any?
|
|
91
|
+
def dead_claimed_count = dead.sum(&:claimed_count)
|
|
92
|
+
def total = nodes.size
|
|
93
|
+
|
|
94
|
+
def find(id)
|
|
95
|
+
nodes.find { |node| node.id == id.to_i }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
def build_nodes
|
|
100
|
+
processes = SolidQueue::Process.order(:kind, :id).to_a
|
|
101
|
+
claimed = SolidQueue::ClaimedExecution.group(:process_id).count
|
|
102
|
+
|
|
103
|
+
processes.map do |process|
|
|
104
|
+
Node.new(process: process, claimed_count: claimed.fetch(process.id, 0), children: [])
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|