pgbus 0.19.4 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -0
  3. data/README.md +96 -7
  4. data/Rakefile +6 -1
  5. data/app/controllers/pgbus/dashboard_controller.rb +2 -0
  6. data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
  7. data/app/controllers/pgbus/workflows_controller.rb +197 -0
  8. data/app/frontend/pgbus/modules/charts.js +130 -4
  9. data/app/frontend/pgbus/modules/charts.min.js +1 -1
  10. data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
  11. data/app/frontend/pgbus/style.css +1 -1
  12. data/app/helpers/pgbus/button_helper.rb +4 -2
  13. data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
  14. data/app/helpers/pgbus/workflows_helper.rb +165 -0
  15. data/app/models/pgbus/workflow_entry.rb +172 -0
  16. data/app/models/pgbus/workflow_event.rb +43 -0
  17. data/app/models/pgbus/workflow_signal.rb +77 -0
  18. data/app/models/pgbus/workflow_step.rb +142 -0
  19. data/app/views/layouts/pgbus/application.html.erb +2 -0
  20. data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
  21. data/app/views/pgbus/workflows/_actions.html.erb +59 -0
  22. data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
  23. data/app/views/pgbus/workflows/_filters.html.erb +29 -0
  24. data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
  25. data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
  26. data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
  27. data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
  28. data/app/views/pgbus/workflows/_timeline.html.erb +39 -0
  29. data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
  30. data/app/views/pgbus/workflows/attention.html.erb +88 -0
  31. data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
  32. data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
  33. data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
  34. data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
  35. data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
  36. data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
  37. data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
  38. data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
  39. data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
  40. data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
  41. data/app/views/pgbus/workflows/index.html.erb +15 -0
  42. data/app/views/pgbus/workflows/not_found.html.erb +5 -0
  43. data/app/views/pgbus/workflows/show.html.erb +93 -0
  44. data/config/i18n-tasks.yml +6 -0
  45. data/config/locales/da.yml +442 -0
  46. data/config/locales/de.yml +442 -0
  47. data/config/locales/en.yml +442 -0
  48. data/config/locales/es.yml +442 -0
  49. data/config/locales/fi.yml +442 -0
  50. data/config/locales/fr.yml +442 -0
  51. data/config/locales/it.yml +442 -0
  52. data/config/locales/ja.yml +442 -0
  53. data/config/locales/nb.yml +442 -0
  54. data/config/locales/nl.yml +442 -0
  55. data/config/locales/pt.yml +442 -0
  56. data/config/locales/sv.yml +442 -0
  57. data/config/routes.rb +26 -0
  58. data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
  59. data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
  60. data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
  61. data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
  62. data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
  63. data/lib/pgbus/active_job/delivery.rb +49 -0
  64. data/lib/pgbus/active_job/executor.rb +3 -1
  65. data/lib/pgbus/configuration/workflow_settings.rb +99 -0
  66. data/lib/pgbus/configuration.rb +21 -3
  67. data/lib/pgbus/generators/migration_detector.rb +40 -0
  68. data/lib/pgbus/instrumentation.rb +32 -0
  69. data/lib/pgbus/integrations/appsignal/dashboard.json +128 -1
  70. data/lib/pgbus/integrations/appsignal/subscriber.rb +16 -1
  71. data/lib/pgbus/mcp/rack_app.rb +7 -2
  72. data/lib/pgbus/mcp/runner.rb +2 -1
  73. data/lib/pgbus/mcp/serialization.rb +25 -0
  74. data/lib/pgbus/mcp/server.rb +53 -10
  75. data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
  76. data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
  77. data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
  78. data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
  79. data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
  80. data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
  81. data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
  82. data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
  83. data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
  84. data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
  85. data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
  86. data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
  87. data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
  88. data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
  89. data/lib/pgbus/mcp/write_tool.rb +134 -0
  90. data/lib/pgbus/mcp.rb +15 -0
  91. data/lib/pgbus/metrics/backends/statsd.rb +6 -0
  92. data/lib/pgbus/metrics/subscriber.rb +14 -1
  93. data/lib/pgbus/metrics/workflow_metrics.rb +65 -0
  94. data/lib/pgbus/process/dispatcher.rb +26 -0
  95. data/lib/pgbus/version.rb +1 -1
  96. data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
  97. data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
  98. data/lib/pgbus/web/data_source/workflow_list.rb +248 -0
  99. data/lib/pgbus/web/data_source.rb +3 -0
  100. data/lib/pgbus/web/workflow_action_plan.rb +62 -0
  101. data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
  102. data/lib/pgbus/web/workflow_insights_report.rb +171 -0
  103. data/lib/pgbus/web/workflow_payload.rb +36 -0
  104. data/lib/pgbus/web/workflow_query.rb +82 -0
  105. data/lib/pgbus/web/workflow_state.rb +76 -0
  106. data/lib/pgbus/web/workflow_timeline.rb +113 -0
  107. data/lib/pgbus/web/workflow_window.rb +51 -0
  108. data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
  109. data/lib/pgbus/workflow/dsl.rb +105 -0
  110. data/lib/pgbus/workflow/error_capture.rb +63 -0
  111. data/lib/pgbus/workflow/exception_policy.rb +170 -0
  112. data/lib/pgbus/workflow/fan_out.rb +157 -0
  113. data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
  114. data/lib/pgbus/workflow/flow_control.rb +94 -0
  115. data/lib/pgbus/workflow/instance.rb +183 -0
  116. data/lib/pgbus/workflow/outside_verbs.rb +138 -0
  117. data/lib/pgbus/workflow/payload.rb +53 -0
  118. data/lib/pgbus/workflow/registry.rb +53 -0
  119. data/lib/pgbus/workflow/removed_step.rb +14 -0
  120. data/lib/pgbus/workflow/runner.rb +218 -0
  121. data/lib/pgbus/workflow/signal_matcher.rb +53 -0
  122. data/lib/pgbus/workflow/signals.rb +132 -0
  123. data/lib/pgbus/workflow/states.rb +119 -0
  124. data/lib/pgbus/workflow/step_context.rb +58 -0
  125. data/lib/pgbus/workflow/step_definition.rb +131 -0
  126. data/lib/pgbus/workflow/step_execution.rb +242 -0
  127. data/lib/pgbus/workflow/step_job.rb +28 -0
  128. data/lib/pgbus/workflow/step_outcome.rb +151 -0
  129. data/lib/pgbus/workflow/steps.rb +103 -0
  130. data/lib/pgbus/workflow/sweep.rb +156 -0
  131. data/lib/pgbus/workflow/test_helpers.rb +152 -0
  132. data/lib/pgbus/workflow/wait_timeout.rb +73 -0
  133. data/lib/pgbus/workflow/waiting.rb +82 -0
  134. data/lib/pgbus/workflow.rb +183 -0
  135. data/sig/pgbus/configuration.rbs +22 -0
  136. metadata +92 -1
@@ -0,0 +1,250 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Web
5
+ # The SQL behind the workflow insights (issue #549): one query per block
6
+ # of the page, each bounded by the window and served by an index #540
7
+ # created. The step stats read idx_pgbus_workflow_steps_insights
8
+ # (workflow_class, step_name, finished_at); the run counts read
9
+ # idx_pgbus_workflows_class_status (workflow_class, status, created_at),
10
+ # with every status listed so the created_at range applies under each;
11
+ # the ended runs read idx_pgbus_workflows_status (status, updated_at): a
12
+ # run ends in the UPDATE that sets its finished_at/cancelled_at, so
13
+ # updated_at >= the ended-at bound keeps every row the window wants.
14
+ #
15
+ # Status lists are the States constants, inlined; every value that comes
16
+ # from outside (a class name, a time, a bucket length) is a bind.
17
+ module WorkflowInsightsQueries
18
+ ALL = Workflow::States::WORKFLOW
19
+ ONGOING = Workflow::States::ONGOING
20
+ # Upper bounds (ms) of the end-to-end duration histogram: < 1 s, 10 s,
21
+ # 1 min, 10 min, 1 h, 6 h, 1 d, 7 d, and the open last bin.
22
+ DURATION_BOUNDS_MS = [1_000, 10_000, 60_000, 600_000, 3_600_000, 21_600_000, 86_400_000, 604_800_000].freeze
23
+ REVISIONS = 5
24
+
25
+ module_function
26
+
27
+ # Every workflow_class with a row, by a loose index scan (one probe per
28
+ # class) instead of a DISTINCT over every run.
29
+ def class_names
30
+ sql = <<~SQL
31
+ WITH RECURSIVE classes AS (
32
+ (SELECT workflow_class FROM #{runs} ORDER BY workflow_class LIMIT 1)
33
+ UNION ALL
34
+ SELECT (SELECT w.workflow_class FROM #{runs} w WHERE w.workflow_class > c.workflow_class
35
+ ORDER BY w.workflow_class LIMIT 1)
36
+ FROM classes c WHERE c.workflow_class IS NOT NULL
37
+ )
38
+ SELECT workflow_class FROM classes WHERE workflow_class IS NOT NULL
39
+ SQL
40
+ select(sql, "Classes").rows.map(&:first)
41
+ end
42
+
43
+ def class_known?(workflow_class)
44
+ select("SELECT 1 FROM #{runs} WHERE workflow_class = $1 LIMIT 1", "Class Known", [workflow_class]).rows.any?
45
+ end
46
+
47
+ # The ongoing runs by (class, status, current step).
48
+ def ongoing(workflow_class: nil)
49
+ sql = <<~SQL
50
+ SELECT workflow_class, status, current_step, COUNT(*)
51
+ FROM #{runs}
52
+ WHERE status IN (#{list(ONGOING)})#{" AND workflow_class = $1" if workflow_class}
53
+ GROUP BY workflow_class, status, current_step
54
+ SQL
55
+ select(sql, "Ongoing", [workflow_class].compact).cast_values.map do |klass, status, step, runs|
56
+ { workflow_class: klass, status: status, step: step, runs: runs.to_i }
57
+ end
58
+ end
59
+
60
+ # Per class: the runs that finished or were cancelled between +since+
61
+ # and +until_time+, and the end-to-end percentiles (created → finished)
62
+ # of the finished.
63
+ def ended(since:, until_time:, workflow_class: nil)
64
+ sql = <<~SQL
65
+ SELECT workflow_class, COUNT(*) FILTER (WHERE status = 'finished'), COUNT(*) FILTER (WHERE status = 'cancelled'),
66
+ COUNT(e2e_ms), #{percentiles("e2e_ms", 0.5, 0.95, 0.99)}
67
+ FROM (#{ended_runs(until_bind: "$2", class_bind: ("$3" if workflow_class))}) ended
68
+ GROUP BY workflow_class
69
+ SQL
70
+ select(sql, "Ended", [since, until_time, workflow_class].compact).cast_values.map do |klass, finished, cancelled, samples, *pcts|
71
+ { workflow_class: klass, finished: finished.to_i, cancelled: cancelled.to_i,
72
+ e2e_ms: percentile_hash(%i[p50 p95 p99], pcts, samples) }
73
+ end
74
+ end
75
+
76
+ # { bin => runs } of one class's finished runs over DURATION_BOUNDS_MS
77
+ # (bin 0 is under the first bound, the last bin is past the last one).
78
+ def duration_histogram(workflow_class, since:, until_time:)
79
+ sql = <<~SQL
80
+ SELECT width_bucket(e2e_ms::float8, ARRAY[#{DURATION_BOUNDS_MS.join(", ")}]::float8[]), COUNT(*)
81
+ FROM (#{ended_runs(until_bind: "$2", class_bind: "$3")}) ended
82
+ WHERE status = 'finished'
83
+ GROUP BY 1
84
+ SQL
85
+ select(sql, "Durations", [since, until_time, workflow_class]).rows.to_h { |bin, runs| [bin.to_i, runs.to_i] }
86
+ end
87
+
88
+ # { bucket start => [started, finished, cancelled] }. Buckets are
89
+ # multiples of +bucket+ seconds on the wall clock of +zone+ (an IANA
90
+ # name), so a day bucket starts at that zone's midnight on both sides
91
+ # of a DST change; a bucket start is that wall clock's epoch seconds
92
+ # (WorkflowInsightsReport.wall_epoch).
93
+ def throughput(workflow_class, since:, until_time:, bucket:, zone: "UTC")
94
+ sql = <<~SQL
95
+ SELECT bucket, SUM(started), SUM(finished), SUM(cancelled) FROM (
96
+ SELECT #{wall_bucket("created_at")} AS bucket, 1 AS started, 0 AS finished, 0 AS cancelled
97
+ FROM #{runs}
98
+ WHERE workflow_class = $2 AND status IN (#{list(ALL)}) AND created_at BETWEEN $1 AND $5
99
+ UNION ALL
100
+ SELECT #{wall_bucket("ended_at")}, 0, (status = 'finished')::int, (status = 'cancelled')::int
101
+ FROM (#{ended_runs(until_bind: "$5", class_bind: "$2")}) ended
102
+ ) buckets
103
+ GROUP BY bucket
104
+ SQL
105
+ select(sql, "Throughput", [since, workflow_class, bucket, zone, until_time]).rows.to_h do |start, started, finished, cancelled|
106
+ [start.to_i * bucket, [started.to_i, finished.to_i, cancelled.to_i]]
107
+ end
108
+ end
109
+
110
+ # The bucket number of a UTC timestamp column on the wall clock of the
111
+ # zone bound as $4, in buckets of $3 seconds.
112
+ def wall_bucket(column)
113
+ "FLOOR(EXTRACT(EPOCH FROM (#{column} AT TIME ZONE 'UTC' AT TIME ZONE $4)) / $3)::bigint"
114
+ end
115
+
116
+ # Per step: the attempts that finished between +since+ and +until_time+.
117
+ def step_stats(workflow_class, since:, until_time:)
118
+ sql = <<~SQL
119
+ SELECT step_name, #{stat_columns}, MIN(created_at)
120
+ FROM (#{attempt_rows}) attempts
121
+ GROUP BY step_name
122
+ SQL
123
+ select(sql, "Steps", [workflow_class, since, until_time]).cast_values.to_h do |name, *stats, first_seen_at|
124
+ [name, attempt_stats(stats).merge(first_seen_at: first_seen_at)]
125
+ end
126
+ end
127
+
128
+ # Per step, over the runs started between +since+ and +until_time+: how many have a row
129
+ # for it (reached) and its mean position in their runs (the step order
130
+ # of a class that no longer loads). Partitioned by the run's bigint id:
131
+ # sorting 50k workflow_id strings under a locale collation costs ~4x.
132
+ def funnel_reached(workflow_class, since:, until_time:)
133
+ sql = <<~SQL
134
+ SELECT step_name, COUNT(*), AVG(position) FROM (
135
+ SELECT s.step_name, RANK() OVER (PARTITION BY w.id ORDER BY MIN(s.id)) AS position
136
+ FROM #{runs} w JOIN #{steps} s ON s.workflow_id = w.workflow_id
137
+ WHERE w.workflow_class = $1 AND w.status IN (#{list(ALL)}) AND w.created_at BETWEEN $2 AND $3
138
+ GROUP BY w.id, s.step_name
139
+ ) reached
140
+ GROUP BY step_name
141
+ SQL
142
+ select(sql, "Funnel", [workflow_class, since, until_time]).rows.to_h do |name, reached, position|
143
+ [name, { reached: reached.to_i, position: position.to_f }]
144
+ end
145
+ end
146
+
147
+ # Where each run started between +since+ and +until_time+ is now:
148
+ # { [step, status] => runs }.
149
+ def funnel_stopped(workflow_class, since:, until_time:)
150
+ sql = <<~SQL
151
+ SELECT current_step, status, COUNT(*)
152
+ FROM #{runs}
153
+ WHERE workflow_class = $1 AND status IN (#{list(ALL)}) AND created_at BETWEEN $2 AND $3
154
+ GROUP BY current_step, status
155
+ SQL
156
+ select(sql, "Stopped", [workflow_class, since, until_time]).rows.to_h { |step, status, runs| [[step, status], runs.to_i] }
157
+ end
158
+
159
+ # The step stats of the REVISIONS app revisions that finished an
160
+ # attempt last, per (revision, step) and per revision (step nil).
161
+ def by_revision(workflow_class, since:, until_time:)
162
+ sql = <<~SQL
163
+ WITH revisions AS (
164
+ SELECT app_revision, MIN(finished_at) AS first_seen_at, MAX(finished_at) AS last_seen_at
165
+ FROM #{steps}
166
+ WHERE workflow_class = $1 AND finished_at BETWEEN $2 AND $3 AND app_revision IS NOT NULL
167
+ GROUP BY app_revision
168
+ ORDER BY last_seen_at DESC
169
+ LIMIT #{REVISIONS}
170
+ )
171
+ SELECT attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at, attempts.step_name, #{stat_columns}
172
+ FROM (#{attempt_rows}) attempts
173
+ JOIN revisions ON revisions.app_revision = attempts.app_revision
174
+ GROUP BY GROUPING SETS ((attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at, attempts.step_name),
175
+ (attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at))
176
+ ORDER BY revisions.last_seen_at DESC, attempts.step_name NULLS FIRST
177
+ SQL
178
+ select(sql, "Revisions", [workflow_class, since, until_time]).cast_values.map do |revision, first_seen, last_seen, name, *stats|
179
+ { revision: revision, first_seen_at: first_seen, last_seen_at: last_seen, step: name, stats: attempt_stats(stats) }
180
+ end
181
+ end
182
+
183
+ # The finished and cancelled runs that ended between $1 and +until_bind+
184
+ # (of the class at +class_bind+): never one dated after now (a skewed
185
+ # clock), which the charts would not show either.
186
+ def ended_runs(until_bind:, class_bind: nil)
187
+ <<~SQL
188
+ SELECT workflow_class, status, COALESCE(finished_at, cancelled_at) AS ended_at,
189
+ CASE WHEN status = 'finished' THEN GREATEST(EXTRACT(EPOCH FROM (finished_at - created_at)) * 1000, 0) END AS e2e_ms
190
+ FROM #{runs}
191
+ WHERE status IN ('finished', 'cancelled') AND updated_at >= $1
192
+ AND COALESCE(finished_at, cancelled_at) BETWEEN $1 AND #{until_bind}#{" AND workflow_class = #{class_bind}" if class_bind}
193
+ SQL
194
+ end
195
+
196
+ # The attempts of class $1 that finished between $2 and $3 (never one
197
+ # dated after now). run_ms: completed or
198
+ # failed attempts that started; pickup_ms (scheduled → started, never
199
+ # negative): every attempt that started.
200
+ def attempt_rows
201
+ <<~SQL
202
+ SELECT step_name, status, outcome, error_class, app_revision, created_at,
203
+ CASE WHEN status IN ('completed', 'failed') AND started_at IS NOT NULL
204
+ THEN GREATEST(EXTRACT(EPOCH FROM (finished_at - started_at)) * 1000, 0) END AS run_ms,
205
+ CASE WHEN started_at IS NOT NULL
206
+ THEN GREATEST(EXTRACT(EPOCH FROM (started_at - scheduled_at)) * 1000, 0) END AS pickup_ms
207
+ FROM #{steps}
208
+ WHERE workflow_class = $1 AND finished_at BETWEEN $2 AND $3
209
+ SQL
210
+ end
211
+
212
+ STAT_KEYS = %i[attempts completed failed reattempted skipped cancelled].freeze
213
+
214
+ def stat_columns
215
+ <<~SQL.squish
216
+ COUNT(*), COUNT(*) FILTER (WHERE status = 'completed'), COUNT(*) FILTER (WHERE error_class IS NOT NULL),
217
+ COUNT(*) FILTER (WHERE outcome = 'reattempted'), COUNT(*) FILTER (WHERE status = 'skipped'),
218
+ COUNT(*) FILTER (WHERE status = 'cancelled'),
219
+ COUNT(run_ms), #{percentiles("run_ms", 0.5, 0.95, 0.99)},
220
+ COUNT(pickup_ms), #{percentiles("pickup_ms", 0.5, 0.95)}
221
+ SQL
222
+ end
223
+
224
+ def attempt_stats(values)
225
+ counts = STAT_KEYS.zip(values.first(STAT_KEYS.size).map(&:to_i)).to_h
226
+ run_samples, run50, run95, run99, pickup_samples, pickup50, pickup95 = values.drop(STAT_KEYS.size)
227
+ counts.merge(run_ms: percentile_hash(%i[p50 p95 p99], [run50, run95, run99], run_samples),
228
+ pickup_ms: percentile_hash(%i[p50 p95], [pickup50, pickup95], pickup_samples))
229
+ end
230
+
231
+ def percentiles(column, *fractions)
232
+ fractions.map { |f| "percentile_cont(#{f}) WITHIN GROUP (ORDER BY #{column})" }.join(", ")
233
+ end
234
+
235
+ def percentile_hash(keys, values, samples)
236
+ keys.zip(values.map { |v| v&.to_f&.round(1) }).to_h.merge(samples: samples.to_i)
237
+ end
238
+
239
+ def list(statuses) = statuses.map { |status| connection.quote(status) }.join(", ")
240
+
241
+ def runs = WorkflowEntry.quoted_table_name
242
+
243
+ def steps = WorkflowStep.quoted_table_name
244
+
245
+ def connection = WorkflowEntry.connection
246
+
247
+ def select(sql, name, binds = []) = connection.exec_query(sql, "WorkflowInsights #{name}", binds)
248
+ end
249
+ end
250
+ end
@@ -0,0 +1,171 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Web
5
+ # Shapes WorkflowInsightsQueries' rows into the hashes the insights pages
6
+ # and MCP tools read (issue #549). Pure: no I/O.
7
+ #
8
+ # Steps come in the class's declaration order (gravestones included, as
9
+ # kind "removed"), then any step name the rows hold that the class does
10
+ # not declare (declared: false). For a class that no longer loads, the
11
+ # names in the rows, ordered by their mean position in the runs started
12
+ # in the window. Live counts are keyed by the run's status (ready,
13
+ # performing, waiting, paused), so each links to the filtered list.
14
+ module WorkflowInsightsReport
15
+ LIVE = Workflow::States::ONGOING.map(&:to_sym).freeze
16
+ STOPPED = Workflow::States::WORKFLOW.map(&:to_sym).freeze
17
+ OUTCOMES = %i[completed failed reattempted skipped cancelled].freeze
18
+ # Percentiles over fewer samples than this get a "few samples" note.
19
+ FEW_SAMPLES = 20
20
+ # The real-time step that walks a window's buckets: 15 min, the finest
21
+ # granularity of a UTC offset change.
22
+ WALK = 900
23
+ DAY = 86_400
24
+
25
+ module_function
26
+
27
+ def summary(name, klass, ongoing_rows, ended_row, attention)
28
+ live = ongoing_rows.sum { |row| row[:runs] }
29
+ e2e = ended_row ? ended_row[:e2e_ms].slice(:p50, :p95, :samples) : { p50: nil, p95: nil, samples: 0 }
30
+ { workflow_class: name, loaded: !klass.nil?, ongoing: live,
31
+ paused: ongoing_rows.select { |row| row[:status] == "paused" }.sum { |row| row[:runs] },
32
+ attention: attention, finished: ended_row&.fetch(:finished) || 0, cancelled: ended_row&.fetch(:cancelled) || 0,
33
+ e2e_ms: e2e, removed_in_flight: removed_in_flight(klass, ongoing_rows) }
34
+ end
35
+
36
+ def detail(name, klass, window:, since:, now:, ongoing:, ended:, attention:, stats:, histogram:, throughput:, reached:,
37
+ stopped:, revisions:, zone: nil)
38
+ names = step_list(klass, stats, ongoing, reached, stopped)
39
+ summary(name, klass, ongoing, ended, attention).merge(
40
+ window: window.to_h.merge(since: since), bucket_seconds: window.bucket_seconds,
41
+ started: stopped.values.sum, e2e_ms: ended ? ended[:e2e_ms] : empty_percentiles(%i[p50 p95 p99]),
42
+ steps: names.map { |entry| step(entry, stats[entry[:name]], ongoing) },
43
+ duration_histogram: duration_histogram(histogram),
44
+ throughput: throughput_buckets(throughput, since: since, now: now, bucket: window.bucket_seconds, zone: zone),
45
+ funnel: names.map { |entry| funnel_step(entry[:name], reached, stopped) },
46
+ by_revision: by_revision(revisions, names.map { |entry| entry[:name] })
47
+ )
48
+ end
49
+
50
+ # [{ name:, kind:, removed:, declared: }] in pipeline order.
51
+ def step_list(klass, stats, ongoing, reached, stopped)
52
+ seen = (stats.keys + ongoing.map { |row| row[:step] } + reached.keys + stopped.keys.map(&:first)).compact.uniq
53
+ declared = klass ? klass.workflow_steps.entries.map { |entry| declared_step(entry) } : []
54
+ extra = (seen - declared.map { |entry| entry[:name] }).sort_by do |name|
55
+ [reached.dig(name, :position) || Float::INFINITY, stats.dig(name, :first_seen_at)&.to_f || Float::INFINITY, name]
56
+ end
57
+ declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false } }
58
+ end
59
+
60
+ def declared_step(entry)
61
+ kind = if entry.removed? then "removed"
62
+ elsif entry.kind == "fan_out" then "fan_out"
63
+ elsif entry.waits_for_signal? then "signal"
64
+ else "step"
65
+ end
66
+ { name: entry.name, kind: kind, removed: entry.removed?, declared: true }
67
+ end
68
+
69
+ def step(entry, stats, ongoing)
70
+ counts = LIVE.to_h { |status| [status, 0] }
71
+ ongoing.each { |row| counts[row[:status].to_sym] += row[:runs] if row[:step] == entry[:name] }
72
+ stats ||= { attempts: 0, run_ms: empty_percentiles(%i[p50 p95 p99]), pickup_ms: empty_percentiles(%i[p50 p95]) }
73
+ entry.merge(counts: counts, sample_size: stats[:attempts], outcomes: OUTCOMES.to_h { |key| [key, stats[key].to_i] },
74
+ rates: rates(stats), run_ms: stats[:run_ms], pickup_ms: stats[:pickup_ms])
75
+ end
76
+
77
+ # Shares of the attempts: failed (an error was recorded), reattempted,
78
+ # skipped. nil without attempts.
79
+ def rates(stats)
80
+ attempts = stats[:attempts].to_i
81
+ %i[failed reattempted skipped].to_h do |key|
82
+ [key, attempts.zero? ? nil : (stats[key].to_f / attempts).round(4)]
83
+ end
84
+ end
85
+
86
+ def empty_percentiles(keys) = keys.to_h { |key| [key, nil] }.merge(samples: 0)
87
+
88
+ # Gravestones some ongoing run still sits on: { "reminder" => 3 }.
89
+ def removed_in_flight(klass, ongoing_rows)
90
+ return {} unless klass
91
+
92
+ names = klass.workflow_steps.removed_names
93
+ ongoing_rows.select { |row| names.include?(row[:step]) }
94
+ .each_with_object(Hash.new(0)) { |row, counts| counts[row[:step]] += row[:runs] }.to_h
95
+ end
96
+
97
+ def duration_histogram(histogram)
98
+ bounds = WorkflowInsightsQueries::DURATION_BOUNDS_MS
99
+ (0..bounds.size).map do |bin|
100
+ { from_ms: bin.zero? ? 0 : bounds[bin - 1], to_ms: bounds[bin], count: histogram.fetch(bin, 0) }
101
+ end
102
+ end
103
+
104
+ # Every bucket of the window, oldest first, empty ones as zeros. +from+
105
+ # and +to+ are the real times its wall-clock start and the next
106
+ # bucket's fall on in +zone+ (a day bucket across a DST change is 23 or
107
+ # 25 hours long; a fall-back hour spans two real hours).
108
+ def throughput_buckets(counts, since:, now:, bucket:, zone: nil)
109
+ zone ||= utc_zone
110
+ bucket_starts(since: since, now: now, bucket: bucket, zone: zone).map do |start|
111
+ started, finished, cancelled = counts.fetch(start, [0, 0, 0])
112
+ { from: from_wall(start, zone), to: from_wall(start + bucket, zone), started: started, finished: finished,
113
+ cancelled: cancelled }
114
+ end
115
+ end
116
+
117
+ # The wall-clock epoch seconds (in +zone+) each bucket from +since+ to
118
+ # +now+ starts at: what WorkflowInsightsQueries.throughput keys by.
119
+ # Walked in real time (steps of at most WALK seconds, finer than any DST
120
+ # change), so a spring-forward night has no bucket for the hour it
121
+ # skips and a fall-back night's repeated hour is one bucket.
122
+ def bucket_starts(since:, now:, bucket:, zone: nil)
123
+ zone ||= utc_zone
124
+ moments = (since.to_i..now.to_i).step([bucket, WALK].min).map { |seconds| Time.at(seconds) } << now
125
+ moments.map { |moment| wall_epoch(moment, zone) / bucket * bucket }.uniq.sort
126
+ end
127
+
128
+ # +time+'s wall clock in +zone+, as epoch seconds.
129
+ def wall_epoch(time, zone)
130
+ local = time.in_time_zone(zone)
131
+ local.to_i + local.utc_offset
132
+ end
133
+
134
+ # The real (UTC) time a bucket's wall-clock start falls on in +zone+:
135
+ # the first of the two in a repeated hour, or, when a DST change skips
136
+ # the start itself (a midnight that jumps to 01:00), the first moment
137
+ # of the bucket that exists.
138
+ def from_wall(wall, zone)
139
+ (0...DAY).step(WALK) do |later|
140
+ return zone.tzinfo.local_to_utc(Time.at(wall + later).utc, true).utc
141
+ rescue TZInfo::PeriodNotFound
142
+ next
143
+ end
144
+ Time.at(wall).utc
145
+ end
146
+
147
+ def utc_zone = ActiveSupport::TimeZone["UTC"]
148
+
149
+ def funnel_step(name, reached, stopped)
150
+ { name: name, reached: reached.dig(name, :reached) || 0,
151
+ stopped: STOPPED.to_h { |status| [status, stopped.fetch([name, status.to_s], 0)] } }
152
+ end
153
+
154
+ # One entry per revision (newest first): its totals and its steps in
155
+ # pipeline order.
156
+ def by_revision(rows, order)
157
+ rows.group_by { |row| row[:revision] }.map do |revision, group|
158
+ total = group.find { |row| row[:step].nil? }
159
+ steps = group.select { |row| row[:step] }.sort_by { |row| order.index(row[:step]) || order.size }
160
+ { revision: revision, first_seen_at: total[:first_seen_at], last_seen_at: total[:last_seen_at],
161
+ **revision_numbers(total[:stats]),
162
+ steps: steps.map { |row| { name: row[:step], **revision_numbers(row[:stats]) } } }
163
+ end
164
+ end
165
+
166
+ def revision_numbers(stats)
167
+ { attempts: stats[:attempts], rates: rates(stats), run_ms: stats[:run_ms].slice(:p50, :p95, :samples) }
168
+ end
169
+ end
170
+ end
171
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Web
5
+ # Workflow arguments, step outputs and signal payloads as the Workflows
6
+ # dashboard and its MCP tools show them (issue #542). They are stored
7
+ # through ActiveJob::Arguments; this reads that stored form without
8
+ # deserializing it, so showing a run never loads its records (a GlobalID
9
+ # shows as its URI) and never runs a custom serializer.
10
+ module WorkflowPayload
11
+ BOOKKEEPING = %w[_aj_symbol_keys _aj_ruby2_keywords _aj_hash_with_indifferent_access].freeze
12
+
13
+ module_function
14
+
15
+ def display(value)
16
+ case value
17
+ when nil then {}
18
+ when Hash then display_hash(value)
19
+ when Array then value.map { |element| element.nil? ? nil : display(element) }
20
+ else value
21
+ end
22
+ end
23
+
24
+ def display_hash(hash)
25
+ return hash["_aj_globalid"] if hash.size == 1 && hash.key?("_aj_globalid")
26
+ if hash.key?("_aj_serialized")
27
+ # A serializer that stores one "value" shows it; one that stores
28
+ # several fields (a Range: begin, end, exclude_end) shows those.
29
+ return hash.key?("value") ? display(hash["value"]) : display_hash(hash.except("_aj_serialized"))
30
+ end
31
+
32
+ hash.except(*BOOKKEEPING).transform_values { |nested| nested.nil? ? nil : display(nested) }
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+ require "time"
5
+ require "active_support/core_ext/object/blank"
6
+ require "active_support/core_ext/time"
7
+
8
+ module Pgbus
9
+ module Web
10
+ # The Workflows list's query string (issue #542), checked: a status from
11
+ # DataSource::WorkflowList::STATUSES, the text filters stripped (blank
12
+ # ones dropped, each at most MAX_LENGTH characters) and the created-at
13
+ # and ended-at (issue #549: finished_at or cancelled_at) ranges parsed in
14
+ # Time.zone (a bare date as the start or the end of that day). Anything
15
+ # else raises Invalid, which the controller answers with 400. #to_param
16
+ # is the same query back, for links that keep it.
17
+ class WorkflowQuery
18
+ class Invalid < StandardError; end
19
+
20
+ TEXT_FILTERS = %i[workflow_class step error_class subject_type subject_id workflow_id].freeze
21
+ DATE_FILTERS = %i[created_from created_to ended_from ended_to].freeze
22
+ MAX_LENGTH = 255
23
+ DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/
24
+ # A date (what <input type="date"> sends) or an ISO 8601 date and time.
25
+ DATE_OR_TIME = /\A\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?\z/
26
+
27
+ attr_reader :status, :filters, :raw
28
+
29
+ def initialize(params)
30
+ @status = (params[:status].presence || "ongoing").to_s
31
+ raise Invalid, "unknown status #{@status.inspect}" unless DataSource::WorkflowList::STATUSES.include?(@status)
32
+
33
+ @raw = {}
34
+ @filters = {}
35
+ TEXT_FILTERS.each { |key| text(key, params[key]) }
36
+ DATE_FILTERS.each { |key| date(key, params[key]) }
37
+ end
38
+
39
+ def filtered? = @raw.any?
40
+
41
+ # The query as URL params (status and the filters as given), merged with +extra+.
42
+ def to_param(**extra) = { status: status, **@raw }.merge(extra).compact
43
+
44
+ private
45
+
46
+ def text(key, value)
47
+ value = scalar(key, value)
48
+ return if value.empty?
49
+ raise Invalid, "#{key} is longer than #{MAX_LENGTH} characters" if value.length > MAX_LENGTH
50
+
51
+ @raw[key] = value
52
+ @filters[key] = value
53
+ end
54
+
55
+ def date(key, value)
56
+ value = scalar(key, value)
57
+ return if value.empty?
58
+
59
+ time = parse_time(value) || raise(Invalid, "#{key} #{value.inspect} is not a date")
60
+ time = time.end_of_day if key.end_with?("_to") && value.match?(DATE_ONLY)
61
+ @raw[key] = value
62
+ @filters[key] = time
63
+ end
64
+
65
+ def scalar(key, value)
66
+ return "" if value.nil?
67
+ raise Invalid, "#{key} must be a single value" unless value.is_a?(String)
68
+
69
+ value.strip
70
+ end
71
+
72
+ # Date.valid_date? because Time.zone.parse rolls 2026-02-30 over to March.
73
+ def parse_time(value)
74
+ return nil unless value.match?(DATE_OR_TIME) && Date.valid_date?(*value[0, 10].split("-").map(&:to_i))
75
+
76
+ Time.zone ? Time.zone.parse(value) : Time.parse(value)
77
+ rescue ArgumentError
78
+ nil
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Web
5
+ # The Workflows dashboard's words for one run (issue #542): the badge
6
+ # (its status, or "scheduled" for a ready run whose step waits for a
7
+ # time still ahead) and what it is doing and what happens next, as a
8
+ # pgbus.workflows.next.<key>_html key and its arguments. Arguments named
9
+ # :step, :signal and :batch are names, :ago a past moment and :time a
10
+ # future one; the helper renders each. Pure.
11
+ module WorkflowState
12
+ BADGE_TONES = { "ready" => :gray, "scheduled" => :gray, "performing" => :blue, "waiting" => :yellow,
13
+ "paused" => :yellow, "finished" => :green, "cancelled" => :red }.freeze
14
+
15
+ Result = Data.define(:badge, :badge_tone, :next_key, :next_args)
16
+
17
+ module_function
18
+
19
+ # row – a DataSource#workflows row. threshold – how long a due step may
20
+ # wait for a worker before it counts as overdue (workflow_stall_threshold).
21
+ def present(row, threshold:, now: Time.current)
22
+ badge, key, args = derive(row, threshold, now)
23
+ Result.new(badge: badge, badge_tone: BADGE_TONES.fetch(badge, :gray), next_key: key, next_args: args)
24
+ end
25
+
26
+ def derive(row, threshold, now)
27
+ step = row[:current_step]
28
+ case row[:status]
29
+ when "performing" then performing(row, step)
30
+ when "waiting" then waiting(row, step)
31
+ when "paused" then paused(row, step)
32
+ when "ready" then ready(row, step, threshold, now)
33
+ when "finished" then ["finished", "finished", { ago: row[:finished_at] || row[:updated_at] }]
34
+ when "cancelled" then ["cancelled", "cancelled", { ago: row[:cancelled_at] || row[:updated_at] }]
35
+ else [row[:status].to_s, "unknown", { status: row[:status].to_s }]
36
+ end
37
+ end
38
+
39
+ def performing(row, step)
40
+ attempt = row[:current_attempt].to_i
41
+ return ["performing", "performing", { step: step }] if attempt <= 1
42
+
43
+ ["performing", "performing_attempt", { step: step, attempt: attempt }]
44
+ end
45
+
46
+ def waiting(row, step)
47
+ if row[:batch_id]
48
+ progress = row[:batch_progress] || {}
49
+ return ["waiting", "waiting_batch", { batch: batch_name(row), done: progress[:done].to_i, total: progress[:total].to_i }]
50
+ end
51
+ return ["waiting", "waiting_step", { step: step }] unless row[:waiting_for]
52
+
53
+ args = { signal: row[:waiting_for], ago: row[:waiting_since] }
54
+ row[:timeout_at] ? ["waiting", "waiting_signal_timeout", args.merge(time: row[:timeout_at])] : ["waiting", "waiting_signal", args]
55
+ end
56
+
57
+ def paused(row, step)
58
+ return ["paused", "paused_error", { step: step, error: row[:error_class] }] if row[:error_class]
59
+
60
+ ["paused", "paused_manual", { step: step }]
61
+ end
62
+
63
+ def ready(row, step, threshold, now)
64
+ due = JobState.time(row[:scheduled_at])
65
+ return ["ready", "ready", { step: step }] unless due
66
+ return ["scheduled", "scheduled", { step: step, time: due }] if due > now
67
+ return ["ready", "overdue", { step: step, ago: due }] if threshold && due < now - threshold
68
+
69
+ ["ready", "due", { step: step, ago: due }]
70
+ end
71
+
72
+ # The fan-out batch as its description names it: "Class#step".
73
+ def batch_name(row) = "#{row[:workflow_class]}##{row[:current_step]}"
74
+ end
75
+ end
76
+ end