flightdeck 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +21 -0
  3. data/README.md +190 -0
  4. data/app/assets/flightdeck/flightdeck-00403457c5d9.css +3 -0
  5. data/app/assets/flightdeck/flightdeck-c02e38040ddb.js +414 -0
  6. data/app/assets/flightdeck/manifest.json +16 -0
  7. data/app/controllers/concerns/flightdeck/toasts.rb +25 -0
  8. data/app/controllers/flightdeck/application_controller.rb +96 -0
  9. data/app/controllers/flightdeck/assets_controller.rb +39 -0
  10. data/app/controllers/flightdeck/jobs/actions_controller.rb +167 -0
  11. data/app/controllers/flightdeck/jobs/discards_controller.rb +44 -0
  12. data/app/controllers/flightdeck/jobs/retries_controller.rb +23 -0
  13. data/app/controllers/flightdeck/jobs_controller.rb +39 -0
  14. data/app/controllers/flightdeck/overview_controller.rb +11 -0
  15. data/app/controllers/flightdeck/processes_controller.rb +49 -0
  16. data/app/controllers/flightdeck/queues_controller.rb +51 -0
  17. data/app/controllers/flightdeck/recurring_tasks_controller.rb +46 -0
  18. data/app/helpers/flightdeck/application_helper.rb +68 -0
  19. data/app/helpers/flightdeck/infrastructure_helper.rb +24 -0
  20. data/app/helpers/flightdeck/jobs_helper.rb +136 -0
  21. data/app/helpers/flightdeck/metrics_helper.rb +49 -0
  22. data/app/models/flightdeck/arguments_preview.rb +95 -0
  23. data/app/models/flightdeck/bulk_action.rb +83 -0
  24. data/app/models/flightdeck/cache.rb +51 -0
  25. data/app/models/flightdeck/cron_schedule.rb +102 -0
  26. data/app/models/flightdeck/error_summary.rb +64 -0
  27. data/app/models/flightdeck/grouped_failures.rb +24 -0
  28. data/app/models/flightdeck/job_detail.rb +163 -0
  29. data/app/models/flightdeck/job_row.rb +93 -0
  30. data/app/models/flightdeck/jobs_query.rb +241 -0
  31. data/app/models/flightdeck/metrics/bar_chart.rb +49 -0
  32. data/app/models/flightdeck/metrics/chart.rb +111 -0
  33. data/app/models/flightdeck/metrics/line_chart.rb +67 -0
  34. data/app/models/flightdeck/metrics/series.rb +165 -0
  35. data/app/models/flightdeck/metrics/sparkline.rb +45 -0
  36. data/app/models/flightdeck/metrics/time_bucket.rb +95 -0
  37. data/app/models/flightdeck/overview.rb +217 -0
  38. data/app/models/flightdeck/process_registry.rb +108 -0
  39. data/app/models/flightdeck/queue_stats.rb +88 -0
  40. data/app/models/flightdeck/recurring_catalog.rb +116 -0
  41. data/app/views/flightdeck/jobs/_failed_row.html.erb +33 -0
  42. data/app/views/flightdeck/jobs/_filter_bar.html.erb +29 -0
  43. data/app/views/flightdeck/jobs/_list.html.erb +89 -0
  44. data/app/views/flightdeck/jobs/_pager.html.erb +22 -0
  45. data/app/views/flightdeck/jobs/_row.html.erb +23 -0
  46. data/app/views/flightdeck/jobs/_state_tabs.html.erb +10 -0
  47. data/app/views/flightdeck/jobs/actions/create.turbo_stream.erb +18 -0
  48. data/app/views/flightdeck/jobs/index.html.erb +13 -0
  49. data/app/views/flightdeck/jobs/show.html.erb +100 -0
  50. data/app/views/flightdeck/metrics/_bar_chart.html.erb +34 -0
  51. data/app/views/flightdeck/metrics/_line_chart.html.erb +38 -0
  52. data/app/views/flightdeck/metrics/_range_control.html.erb +7 -0
  53. data/app/views/flightdeck/metrics/_sparkline.html.erb +9 -0
  54. data/app/views/flightdeck/overview/_completion.html.erb +19 -0
  55. data/app/views/flightdeck/overview/_failures.html.erb +28 -0
  56. data/app/views/flightdeck/overview/_fleet.html.erb +21 -0
  57. data/app/views/flightdeck/overview/_queues.html.erb +41 -0
  58. data/app/views/flightdeck/overview/_throughput.html.erb +23 -0
  59. data/app/views/flightdeck/overview/_tiles.html.erb +14 -0
  60. data/app/views/flightdeck/overview/index.html.erb +54 -0
  61. data/app/views/flightdeck/processes/_fleet.html.erb +54 -0
  62. data/app/views/flightdeck/processes/_row.html.erb +28 -0
  63. data/app/views/flightdeck/processes/index.html.erb +9 -0
  64. data/app/views/flightdeck/processes/prune.turbo_stream.erb +15 -0
  65. data/app/views/flightdeck/queues/_cards.html.erb +46 -0
  66. data/app/views/flightdeck/queues/index.html.erb +9 -0
  67. data/app/views/flightdeck/queues/update.turbo_stream.erb +15 -0
  68. data/app/views/flightdeck/recurring_tasks/_table.html.erb +53 -0
  69. data/app/views/flightdeck/recurring_tasks/index.html.erb +9 -0
  70. data/app/views/flightdeck/recurring_tasks/run.turbo_stream.erb +15 -0
  71. data/app/views/flightdeck/shared/_toast.html.erb +7 -0
  72. data/app/views/layouts/flightdeck/application.html.erb +110 -0
  73. data/config/routes.rb +40 -0
  74. data/lib/flightdeck/assets.rb +78 -0
  75. data/lib/flightdeck/configuration.rb +80 -0
  76. data/lib/flightdeck/engine.rb +31 -0
  77. data/lib/flightdeck/version.rb +5 -0
  78. data/lib/flightdeck.rb +31 -0
  79. metadata +150 -0
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flightdeck
4
+ module Metrics
5
+ # Average completion time: area under a line, with gaps where no jobs
6
+ # finished at all.
7
+ class LineChart < Chart
8
+ DOT_RADIUS = 3
9
+
10
+ def coordinates
11
+ @coordinates ||= points.each_with_index.map do |point, index|
12
+ next nil if point.seconds.nil?
13
+
14
+ { x: round(plot_left + (slot_width * index) + (slot_width / 2.0)),
15
+ y: round(y_for(point.seconds)),
16
+ seconds: point.seconds,
17
+ title: point_title(point) }
18
+ end
19
+ end
20
+
21
+ # Contiguous runs of buckets that had data. Drawing them separately keeps
22
+ # a gap in the data looking like a gap, instead of a straight line across
23
+ # a period we know nothing about.
24
+ def segments
25
+ coordinates.chunk_while { |a, b| !a.nil? && !b.nil? }.filter_map do |chunk|
26
+ present = chunk.compact
27
+ present.size.positive? ? present : nil
28
+ end
29
+ end
30
+
31
+ def line_path(segment)
32
+ return "" if segment.empty?
33
+
34
+ segment.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
35
+ end
36
+
37
+ def area_path(segment)
38
+ return "" if segment.empty?
39
+
40
+ "#{line_path(segment)} L #{segment.last[:x]} #{plot_bottom} L #{segment.first[:x]} #{plot_bottom} Z"
41
+ end
42
+
43
+ def markers = coordinates.compact
44
+
45
+ def last_marker = markers.last
46
+
47
+ def any_data? = markers.any?
48
+
49
+ private
50
+ def raw_max = points.filter_map(&:seconds).max.to_f
51
+
52
+ def format_value(value)
53
+ case value
54
+ when 0 then "0s"
55
+ when 0...1 then "#{(value * 1000).round}ms"
56
+ when 1...60 then "#{value.round(1).to_s.sub(/\.0\z/, "")}s"
57
+ when 60...3600 then "#{(value / 60).round}m"
58
+ else "#{(value / 3600).round(1)}h"
59
+ end
60
+ end
61
+
62
+ def point_title(point)
63
+ "#{full_time(point.at)} · avg #{format_value(point.seconds)} to completion"
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flightdeck
4
+ module Metrics
5
+ # Time series for the Overview charts, computed live from the Solid Queue
6
+ # tables — no rollup tables, no background aggregation.
7
+ #
8
+ # Every series is gap-filled in Ruby so a chart always has a complete axis:
9
+ # a bucket with no rows is a real zero (or a real "no data"), not a missing
10
+ # point that would make the chart lie about its own shape.
11
+ class Series
12
+ Window = Struct.new(:key, :label, :duration, :bucket_seconds, :subtitle, :tick_every,
13
+ :tick_format, keyword_init: true) do
14
+ def bucket_count = (duration / bucket_seconds).to_i
15
+ end
16
+
17
+ WINDOWS = {
18
+ "1h" => Window.new(key: "1h", label: "1H", duration: 1.hour, bucket_seconds: 5.minutes.to_i,
19
+ subtitle: "per 5 minutes", tick_every: 3, tick_format: "%H:%M"),
20
+ "24h" => Window.new(key: "24h", label: "24H", duration: 24.hours, bucket_seconds: 1.hour.to_i,
21
+ subtitle: "per hour", tick_every: 4, tick_format: "%H:%M"),
22
+ "7d" => Window.new(key: "7d", label: "7D", duration: 7.days, bucket_seconds: 6.hours.to_i,
23
+ subtitle: "per 6 hours", tick_every: 4, tick_format: "%-m/%-d")
24
+ }.freeze
25
+
26
+ DEFAULT_WINDOW = "24h"
27
+ SPARKLINE_HOURS = 16
28
+ SPARKLINE_BUCKET = 1.hour.to_i
29
+
30
+ Point = Struct.new(:at, :succeeded, :failed, :seconds, keyword_init: true) do
31
+ def total = succeeded.to_i + failed.to_i
32
+ def blank? = seconds.nil?
33
+ end
34
+
35
+ attr_reader :window, :now
36
+
37
+ def self.window_for(key)
38
+ WINDOWS.fetch(key.to_s, WINDOWS.fetch(DEFAULT_WINDOW))
39
+ end
40
+
41
+ def initialize(window: DEFAULT_WINDOW, now: Time.current)
42
+ @window = window.is_a?(Window) ? window : self.class.window_for(window)
43
+ @now = now.utc
44
+ end
45
+
46
+ # Bucket start times, aligned to absolute epoch multiples so that the same
47
+ # bucket has the same boundaries in every process and every request.
48
+ def buckets
49
+ @buckets ||= begin
50
+ last = align(@now.to_i)
51
+ count = window.bucket_count
52
+ Array.new(count) { |i| last - ((count - 1 - i) * window.bucket_seconds) }
53
+ end
54
+ end
55
+
56
+ def starts_at = Time.at(buckets.first).utc
57
+ def ends_at = Time.at(buckets.last + window.bucket_seconds).utc
58
+
59
+ # Jobs finished (succeeded) and executions failed, per bucket.
60
+ #
61
+ # A job that failed has no finished_at — Solid Queue leaves it null and
62
+ # records a failed_executions row — so the two series never double count.
63
+ def throughput
64
+ cached(:throughput) do
65
+ succeeded = grouped_count(SolidQueue::Job.where(finished_at: starts_at..), :finished_at)
66
+ failed = grouped_count(SolidQueue::FailedExecution.where(created_at: starts_at..), :created_at,
67
+ table: SolidQueue::FailedExecution.table_name)
68
+
69
+ buckets.map do |bucket|
70
+ Point.new(at: Time.at(bucket).utc,
71
+ succeeded: succeeded.fetch(bucket, 0),
72
+ failed: failed.fetch(bucket, 0))
73
+ end
74
+ end
75
+ end
76
+
77
+ # Average wall-clock time from enqueue to finish, per bucket.
78
+ #
79
+ # This is time *to completion*, not time to start: Solid Queue deletes the
80
+ # claimed_executions row when a job finishes, so the moment a job actually
81
+ # started is not recoverable after the fact and no honest historical
82
+ # time-to-start series can be built from these tables.
83
+ def completion_time
84
+ cached(:completion_time) do
85
+ averages = SolidQueue::Job
86
+ .where(finished_at: starts_at..)
87
+ .group(TimeBucket.sql("#{jobs_table}.finished_at", window.bucket_seconds))
88
+ .average(TimeBucket.elapsed_sql("#{jobs_table}.created_at", "#{jobs_table}.finished_at"))
89
+ .transform_keys(&:to_i)
90
+
91
+ buckets.map do |bucket|
92
+ value = averages[bucket]
93
+ Point.new(at: Time.at(bucket).utc, seconds: value&.to_f)
94
+ end
95
+ end
96
+ end
97
+
98
+ def total_succeeded = throughput.sum(&:succeeded)
99
+ def total_failed = throughput.sum(&:failed)
100
+
101
+ def empty? = throughput.all? { |point| point.total.zero? }
102
+
103
+ # Finished-job retention shorter than the window means the oldest buckets
104
+ # are missing rows that really did happen. The chart says so rather than
105
+ # showing a decline that is an artefact of purging.
106
+ def truncated_by_retention?
107
+ retention.present? && retention < window.duration
108
+ end
109
+
110
+ def retention
111
+ return nil unless SolidQueue.respond_to?(:clear_finished_jobs_after)
112
+ return nil unless SolidQueue.preserve_finished_jobs?
113
+
114
+ SolidQueue.clear_finished_jobs_after
115
+ end
116
+
117
+ # One query for every queue's hourly finished count, fanned out in Ruby.
118
+ # Used by the queue cards and the Overview mini-table.
119
+ def self.queue_sparklines(now: Time.current, hours: SPARKLINE_HOURS)
120
+ Flightdeck::Cache.fetch("sparklines", hours, align_to(now.to_i, SPARKLINE_BUCKET),
121
+ expires_in: Flightdeck.config.chart_cache_ttl) do
122
+ last = align_to(now.to_i, SPARKLINE_BUCKET)
123
+ slots = Array.new(hours) { |i| last - ((hours - 1 - i) * SPARKLINE_BUCKET) }
124
+
125
+ counts = SolidQueue::Job
126
+ .where(finished_at: Time.at(slots.first).utc..)
127
+ .group(:queue_name)
128
+ .group(TimeBucket.sql("#{SolidQueue::Job.table_name}.finished_at", SPARKLINE_BUCKET))
129
+ .count
130
+
131
+ fanned = Hash.new { |hash, key| hash[key] = Array.new(hours, 0) }
132
+ counts.each do |(queue_name, bucket), count|
133
+ index = slots.index(bucket.to_i)
134
+ next unless index
135
+
136
+ fanned[queue_name][index] = count
137
+ end
138
+ fanned.default = nil
139
+ fanned
140
+ end
141
+ end
142
+
143
+ def self.align_to(epoch, seconds) = (epoch / seconds) * seconds
144
+
145
+ private
146
+ def jobs_table = SolidQueue::Job.table_name
147
+
148
+ def align(epoch) = self.class.align_to(epoch, window.bucket_seconds)
149
+
150
+ def grouped_count(relation, column, table: SolidQueue::Job.table_name)
151
+ relation
152
+ .group(TimeBucket.sql("#{table}.#{column}", window.bucket_seconds))
153
+ .count
154
+ .transform_keys(&:to_i)
155
+ end
156
+
157
+ # Keyed on the aligned bucket so the cache turns over with the buckets
158
+ # themselves rather than at an arbitrary moment.
159
+ def cached(name, &block)
160
+ Flightdeck::Cache.fetch("series", name, window.key, buckets.last,
161
+ expires_in: Flightdeck.config.chart_cache_ttl, &block)
162
+ end
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flightdeck
4
+ module Metrics
5
+ # Tiny area+line for the queue cards and the Overview mini-table.
6
+ class Sparkline
7
+ WIDTH = 100
8
+ HEIGHT = 30
9
+
10
+ attr_reader :values
11
+
12
+ def initialize(values)
13
+ @values = Array(values).map(&:to_i)
14
+ end
15
+
16
+ def width = WIDTH
17
+ def height = HEIGHT
18
+ def any? = values.any? && values.max.positive?
19
+
20
+ def coordinates
21
+ @coordinates ||= begin
22
+ max = [ values.max.to_i, 1 ].max
23
+ step = values.size > 1 ? WIDTH.to_f / (values.size - 1) : WIDTH.to_f
24
+
25
+ values.each_with_index.map do |value, index|
26
+ { x: (index * step).round(2),
27
+ y: (HEIGHT - 2 - ((value.to_f / max) * (HEIGHT - 4))).round(2) }
28
+ end
29
+ end
30
+ end
31
+
32
+ def line_path
33
+ coordinates.each_with_index.map { |c, i| "#{i.zero? ? "M" : "L"} #{c[:x]} #{c[:y]}" }.join(" ")
34
+ end
35
+
36
+ def area_path
37
+ return "" if coordinates.empty?
38
+
39
+ "#{line_path} L #{coordinates.last[:x]} #{HEIGHT} L #{coordinates.first[:x]} #{HEIGHT} Z"
40
+ end
41
+
42
+ def last_point = coordinates.last
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flightdeck
4
+ module Metrics
5
+ # The only adapter-specific SQL in Flightdeck.
6
+ #
7
+ # Charts bucket timestamps into fixed-width windows. Doing that portably
8
+ # means converting a timestamp to a UTC epoch integer and dividing — never
9
+ # date_trunc, which does not exist everywhere and drags the server's
10
+ # timezone into the answer. Everything downstream is an integer number of
11
+ # seconds since the epoch, in UTC; the display timezone is applied at render
12
+ # time and nowhere else.
13
+ module TimeBucket
14
+ class UnsupportedAdapter < StandardError; end
15
+
16
+ # Identifiers are built by Flightdeck, never by a request, but they are
17
+ # still validated before being interpolated so that can never change by
18
+ # accident.
19
+ IDENTIFIER = /\A[a-z_][a-z0-9_]*(\.[a-z_][a-z0-9_]*)?\z/
20
+
21
+ class << self
22
+ def adapter_name
23
+ SolidQueue::Record.connection.adapter_name.to_s
24
+ end
25
+
26
+ # Normalises the adapter into the three SQL dialects we support.
27
+ def family(name = adapter_name)
28
+ case name.to_s.downcase
29
+ when /postgres/ then :postgresql
30
+ when /mysql|trilogy/ then :mysql
31
+ when /sqlite/ then :sqlite
32
+ else
33
+ raise UnsupportedAdapter,
34
+ "Flightdeck cannot bucket time on the #{name} adapter. " \
35
+ "Supported adapters: PostgreSQL, MySQL/Trilogy, SQLite."
36
+ end
37
+ end
38
+
39
+ # Seconds since the UTC epoch, as a number the database can do
40
+ # arithmetic on.
41
+ def epoch(column, family: self.family)
42
+ col = identifier!(column)
43
+
44
+ case family
45
+ when :postgresql then "EXTRACT(EPOCH FROM #{col})"
46
+ when :mysql then "UNIX_TIMESTAMP(#{col})"
47
+ when :sqlite then "CAST(strftime('%s', #{col}) AS INTEGER)"
48
+ else raise UnsupportedAdapter, "unknown family #{family.inspect}"
49
+ end
50
+ end
51
+
52
+ # The epoch second at which `column`'s bucket starts.
53
+ #
54
+ # SQLite gets integer division rather than FLOOR() on purpose: floor() is
55
+ # only available when SQLite is compiled with math functions, and
56
+ # integer division is exact for the post-1970 timestamps we deal with.
57
+ def bucket(column, seconds, family: self.family)
58
+ seconds = interval!(seconds)
59
+ col = identifier!(column)
60
+
61
+ case family
62
+ when :postgresql then "FLOOR(EXTRACT(EPOCH FROM #{col}) / #{seconds}) * #{seconds}"
63
+ when :mysql then "FLOOR(UNIX_TIMESTAMP(#{col}) / #{seconds}) * #{seconds}"
64
+ when :sqlite then "(CAST(strftime('%s', #{col}) AS INTEGER) / #{seconds}) * #{seconds}"
65
+ else raise UnsupportedAdapter, "unknown family #{family.inspect}"
66
+ end
67
+ end
68
+
69
+ def sql(column, seconds) = Arel.sql(bucket(column, seconds))
70
+
71
+ # Seconds between two timestamp columns, for averaging in SQL.
72
+ def elapsed(from_column, to_column, family: self.family)
73
+ "(#{epoch(to_column, family: family)} - #{epoch(from_column, family: family)})"
74
+ end
75
+
76
+ def elapsed_sql(from_column, to_column) = Arel.sql(elapsed(from_column, to_column))
77
+
78
+ private
79
+ def identifier!(column)
80
+ value = column.to_s
81
+ return value if IDENTIFIER.match?(value)
82
+
83
+ raise ArgumentError, "#{column.inspect} is not a plain column identifier"
84
+ end
85
+
86
+ def interval!(seconds)
87
+ value = Integer(seconds)
88
+ return value if value.positive?
89
+
90
+ raise ArgumentError, "bucket width must be a positive number of seconds"
91
+ end
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,217 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flightdeck
4
+ # Everything the Overview page shows, assembled from the same query objects
5
+ # the rest of the dashboard uses.
6
+ class Overview
7
+ WINDOW = 24.hours
8
+ RECENT_FAILURES = 5
9
+ TOP_QUEUES = 6
10
+
11
+ # trend is :up / :down / :flat and says which way the number moved, while
12
+ # `good` says whether that movement is a good thing — more throughput is
13
+ # good, more failures are not.
14
+ Tile = Struct.new(:label, :value, :unit, :detail, :trend, :good, keyword_init: true)
15
+
16
+ attr_reader :now
17
+
18
+ def initialize(now: Time.current)
19
+ @now = now
20
+ end
21
+
22
+ def tiles
23
+ [ processed_tile, failed_tile, ready_tile, scheduled_tile, in_progress_tile, oldest_ready_tile ]
24
+ end
25
+
26
+ def series(window: Metrics::Series::DEFAULT_WINDOW)
27
+ @series ||= {}
28
+ @series[window.to_s] ||= Metrics::Series.new(window: window, now: now)
29
+ end
30
+
31
+ def queues
32
+ @queues ||= QueueStats.new.rows.sort_by { |row| -row.depth }.first(TOP_QUEUES)
33
+ end
34
+
35
+ def max_queue_depth
36
+ @max_queue_depth ||= [ queues.map(&:depth).max.to_i, 1 ].max
37
+ end
38
+
39
+ def sparklines
40
+ @sparklines ||= Metrics::Series.queue_sparklines(now: now)
41
+ end
42
+
43
+ def sparkline_for(queue_name)
44
+ Metrics::Sparkline.new(sparklines[queue_name] || [])
45
+ end
46
+
47
+ def recent_failures
48
+ @recent_failures ||= JobsQuery.new(state: :failed, limit: RECENT_FAILURES).rows
49
+ end
50
+
51
+ def failed_count
52
+ @failed_count ||= JobsQuery.new(state: :failed).count
53
+ end
54
+
55
+ def registry
56
+ @registry ||= ProcessRegistry.new
57
+ end
58
+
59
+ # --- counts ---------------------------------------------------------------
60
+
61
+ def processed_24h
62
+ @processed_24h ||= counted("processed", WINDOW) { SolidQueue::Job.where(finished_at: (now - WINDOW)..now) }
63
+ end
64
+
65
+ def processed_prior_24h
66
+ @processed_prior_24h ||= counted("processed_prior", WINDOW) do
67
+ SolidQueue::Job.where(finished_at: (now - (WINDOW * 2))...(now - WINDOW))
68
+ end
69
+ end
70
+
71
+ def failed_24h
72
+ @failed_24h ||= counted("failed", WINDOW) { SolidQueue::FailedExecution.where(created_at: (now - WINDOW)..now) }
73
+ end
74
+
75
+ def ready_now = @ready_now ||= counted("ready") { SolidQueue::ReadyExecution.all }
76
+ def scheduled = @scheduled ||= counted("scheduled") { SolidQueue::ScheduledExecution.all }
77
+ def in_progress = @in_progress ||= counted("in_progress") { SolidQueue::ClaimedExecution.all }
78
+
79
+ def next_scheduled_at
80
+ @next_scheduled_at ||= Cache.fetch("overview", "next_scheduled", expires_in: count_ttl) do
81
+ SolidQueue::ScheduledExecution.minimum(:scheduled_at)
82
+ end
83
+ end
84
+
85
+ def oldest_ready_at
86
+ @oldest_ready_at ||= Cache.fetch("overview", "oldest_ready", expires_in: count_ttl) do
87
+ SolidQueue::ReadyExecution.minimum(:created_at)
88
+ end
89
+ end
90
+
91
+ def oldest_ready_age
92
+ oldest_ready_at && now - oldest_ready_at
93
+ end
94
+
95
+ # Total worker capacity, summed from each worker's registered thread pool
96
+ # size. Returns nil when no worker reported one, so the tile can omit the
97
+ # denominator rather than divide by a number it invented.
98
+ #
99
+ # A worker that has stopped sending heartbeats is not capacity, whatever its
100
+ # metadata still says, so dead workers are excluded — otherwise utilization
101
+ # reads low precisely when the fleet is in trouble.
102
+ def slots
103
+ @slots ||= begin
104
+ sizes = registry.nodes.filter_map do |node|
105
+ next unless node.claims_executions? && !node.dead?
106
+
107
+ Integer(node.metadata.symbolize_keys[:thread_pool_size], exception: false)
108
+ end
109
+ sizes.any? ? sizes.sum : nil
110
+ end
111
+ end
112
+
113
+ def utilization
114
+ return nil if slots.nil? || slots.zero?
115
+
116
+ (in_progress.to_f / slots * 100).round
117
+ end
118
+
119
+ def failure_rate
120
+ total = processed_24h + failed_24h
121
+ return nil if total.zero?
122
+
123
+ (failed_24h.to_f / total * 100).round(1)
124
+ end
125
+
126
+ def processed_delta
127
+ return nil if processed_prior_24h.zero?
128
+
129
+ ((processed_24h - processed_prior_24h).to_f / processed_prior_24h * 100).round(1)
130
+ end
131
+
132
+ private
133
+ def count_ttl = Flightdeck.config.poll_interval
134
+
135
+ def counted(name, window = nil, &relation)
136
+ Cache.fetch("overview", name, window&.to_i, expires_in: count_ttl) do
137
+ relation.call.limit(Flightdeck.config.count_cap).count
138
+ end
139
+ end
140
+
141
+ def processed_tile
142
+ delta = processed_delta
143
+
144
+ Tile.new(
145
+ label: "Processed · 24h",
146
+ value: processed_24h,
147
+ detail: delta.nil? ? "no prior window to compare" : "#{delta.abs}% vs prior 24h",
148
+ trend: trend_for(delta),
149
+ good: delta.nil? || delta >= 0
150
+ )
151
+ end
152
+
153
+ def failed_tile
154
+ rate = failure_rate
155
+
156
+ Tile.new(
157
+ label: "Failed · 24h",
158
+ value: failed_24h,
159
+ detail: rate.nil? ? "nothing finished yet" : "#{rate}% failure rate",
160
+ trend: failed_24h.positive? ? :up : :flat,
161
+ good: failed_24h.zero?
162
+ )
163
+ end
164
+
165
+ def ready_tile
166
+ Tile.new(label: "Ready now", value: ready_now, detail: "waiting to be claimed", trend: :flat, good: true)
167
+ end
168
+
169
+ def scheduled_tile
170
+ detail =
171
+ if next_scheduled_at.nil? then "nothing scheduled"
172
+ elsif next_scheduled_at <= now then "due now"
173
+ else "next due in #{humanize_duration(next_scheduled_at - now)}"
174
+ end
175
+
176
+ Tile.new(label: "Scheduled", value: scheduled, detail: detail, trend: :flat, good: true)
177
+ end
178
+
179
+ def in_progress_tile
180
+ Tile.new(
181
+ label: "In progress",
182
+ value: in_progress,
183
+ unit: slots ? "/ #{ActiveSupport::NumberHelper.number_to_delimited(slots)} slots" : nil,
184
+ detail: utilization ? "#{utilization}% utilization" : "no worker capacity reported",
185
+ trend: :flat,
186
+ good: true
187
+ )
188
+ end
189
+
190
+ def oldest_ready_tile
191
+ Tile.new(
192
+ label: "Oldest ready",
193
+ value: oldest_ready_age ? humanize_duration(oldest_ready_age) : "—",
194
+ detail: oldest_ready_age ? "longest a job has waited" : "nothing waiting",
195
+ trend: :flat,
196
+ good: true
197
+ )
198
+ end
199
+
200
+ def trend_for(delta)
201
+ return :flat if delta.nil? || delta.abs < 0.05
202
+
203
+ delta.positive? ? :up : :down
204
+ end
205
+
206
+ def humanize_duration(seconds)
207
+ seconds = seconds.to_f.abs
208
+ case seconds
209
+ when 0...1 then "#{(seconds * 1000).round}ms"
210
+ when 1...60 then "#{seconds.round(1).to_s.sub(/\.0\z/, "")}s"
211
+ when 60...3600 then "#{(seconds / 60).round}m"
212
+ when 3600...86_400 then "#{(seconds / 3600).round(1)}h"
213
+ else "#{(seconds / 86_400).round(1)}d"
214
+ end
215
+ end
216
+ end
217
+ end
@@ -0,0 +1,108 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flightdeck
4
+ # The Solid Queue fleet: every registered process, grouped supervisor →
5
+ # children, with heartbeat freshness and claimed counts.
6
+ #
7
+ # Two queries total, regardless of fleet size.
8
+ class ProcessRegistry
9
+ # Freshness is measured against Solid Queue's own liveness threshold rather
10
+ # than a number of our own, so what Flightdeck calls dead is exactly what
11
+ # the pruner will collect.
12
+ FRESH_FRACTION = 0.2
13
+
14
+ Node = Struct.new(:process, :claimed_count, :children, keyword_init: true) do
15
+ def id = process.id
16
+ def kind = process.kind
17
+ def name = process.name
18
+ def hostname = process.hostname
19
+ def pid = process.pid
20
+ def last_heartbeat_at = process.last_heartbeat_at
21
+ def supervisor? = kind == "Supervisor"
22
+ def claims_executions? = kind == "Worker"
23
+
24
+ def heartbeat_age
25
+ Time.current - last_heartbeat_at
26
+ end
27
+
28
+ def freshness
29
+ threshold = SolidQueue.process_alive_threshold
30
+ age = heartbeat_age
31
+
32
+ if age >= threshold then :dead
33
+ elsif age >= threshold * FRESH_FRACTION then :stale
34
+ else :fresh
35
+ end
36
+ end
37
+
38
+ def dead? = freshness == :dead
39
+ def prunable? = dead?
40
+
41
+ def metadata
42
+ process.metadata.presence || {}
43
+ end
44
+
45
+ # Human summary of what this process was configured to do. Keys differ per
46
+ # process kind, so known ones are formatted and anything else is shown
47
+ # verbatim rather than dropped.
48
+ def config_summary
49
+ parts = []
50
+ data = metadata.symbolize_keys
51
+
52
+ parts << "queues: #{data[:queues]}" if data[:queues].present?
53
+ parts << "#{data[:thread_pool_size]} threads" if data[:thread_pool_size].present?
54
+ parts << "batch #{data[:batch_size]}" if data[:batch_size].present?
55
+ parts << "every #{data[:polling_interval]}s" if data[:polling_interval].present?
56
+ parts << "#{Array(data[:recurring_schedule]).size} recurring tasks" if data[:recurring_schedule].present?
57
+
58
+ known = %i[queues thread_pool_size batch_size polling_interval recurring_schedule
59
+ concurrency_maintenance_interval]
60
+ data.except(*known).each { |key, value| parts << "#{key}: #{value}" }
61
+
62
+ parts.join(" · ")
63
+ end
64
+ end
65
+
66
+ def self.all = new
67
+
68
+ def nodes
69
+ @nodes ||= build_nodes
70
+ end
71
+
72
+ # Supervisors with their children nested; anything unsupervised (or whose
73
+ # supervisor row has gone) is listed flat so it can never be hidden.
74
+ def tree
75
+ @tree ||= begin
76
+ by_supervisor = nodes.group_by { |node| node.process.supervisor_id }
77
+ known_ids = nodes.map(&:id).to_set
78
+
79
+ roots = nodes.select { |node| node.process.supervisor_id.nil? || !known_ids.include?(node.process.supervisor_id) }
80
+ roots.each { |root| root.children = by_supervisor.fetch(root.id, []) }
81
+ roots
82
+ end
83
+ end
84
+
85
+ def flattened
86
+ tree.flat_map { |root| [ root, *root.children ] }
87
+ end
88
+
89
+ def dead = nodes.select(&:dead?)
90
+ def any_dead? = dead.any?
91
+ def dead_claimed_count = dead.sum(&:claimed_count)
92
+ def total = nodes.size
93
+
94
+ def find(id)
95
+ nodes.find { |node| node.id == id.to_i }
96
+ end
97
+
98
+ private
99
+ def build_nodes
100
+ processes = SolidQueue::Process.order(:kind, :id).to_a
101
+ claimed = SolidQueue::ClaimedExecution.group(:process_id).count
102
+
103
+ processes.map do |process|
104
+ Node.new(process: process, claimed_count: claimed.fetch(process.id, 0), children: [])
105
+ end
106
+ end
107
+ end
108
+ end