pgbus 0.19.3 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/README.md +104 -9
  4. data/Rakefile +6 -1
  5. data/app/controllers/pgbus/dashboard_controller.rb +2 -0
  6. data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
  7. data/app/controllers/pgbus/workflows_controller.rb +197 -0
  8. data/app/frontend/pgbus/modules/charts.js +130 -4
  9. data/app/frontend/pgbus/modules/charts.min.js +1 -1
  10. data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
  11. data/app/frontend/pgbus/style.css +1 -1
  12. data/app/helpers/pgbus/button_helper.rb +4 -2
  13. data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
  14. data/app/helpers/pgbus/workflows_helper.rb +165 -0
  15. data/app/models/pgbus/workflow_entry.rb +172 -0
  16. data/app/models/pgbus/workflow_event.rb +43 -0
  17. data/app/models/pgbus/workflow_signal.rb +77 -0
  18. data/app/models/pgbus/workflow_step.rb +142 -0
  19. data/app/views/layouts/pgbus/application.html.erb +2 -0
  20. data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
  21. data/app/views/pgbus/workflows/_actions.html.erb +59 -0
  22. data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
  23. data/app/views/pgbus/workflows/_filters.html.erb +29 -0
  24. data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
  25. data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
  26. data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
  27. data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
  28. data/app/views/pgbus/workflows/_timeline.html.erb +39 -0
  29. data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
  30. data/app/views/pgbus/workflows/attention.html.erb +88 -0
  31. data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
  32. data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
  33. data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
  34. data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
  35. data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
  36. data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
  37. data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
  38. data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
  39. data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
  40. data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
  41. data/app/views/pgbus/workflows/index.html.erb +15 -0
  42. data/app/views/pgbus/workflows/not_found.html.erb +5 -0
  43. data/app/views/pgbus/workflows/show.html.erb +93 -0
  44. data/config/i18n-tasks.yml +6 -0
  45. data/config/locales/da.yml +442 -0
  46. data/config/locales/de.yml +442 -0
  47. data/config/locales/en.yml +442 -0
  48. data/config/locales/es.yml +442 -0
  49. data/config/locales/fi.yml +442 -0
  50. data/config/locales/fr.yml +442 -0
  51. data/config/locales/it.yml +442 -0
  52. data/config/locales/ja.yml +442 -0
  53. data/config/locales/nb.yml +442 -0
  54. data/config/locales/nl.yml +442 -0
  55. data/config/locales/pt.yml +442 -0
  56. data/config/locales/sv.yml +442 -0
  57. data/config/routes.rb +26 -0
  58. data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
  59. data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
  60. data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
  61. data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
  62. data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
  63. data/lib/pgbus/active_job/adapter.rb +67 -26
  64. data/lib/pgbus/active_job/batch_id.rb +23 -1
  65. data/lib/pgbus/active_job/delivery.rb +49 -0
  66. data/lib/pgbus/active_job/executor.rb +3 -1
  67. data/lib/pgbus/batch/builder.rb +1 -0
  68. data/lib/pgbus/batch.rb +35 -3
  69. data/lib/pgbus/configuration/workflow_settings.rb +99 -0
  70. data/lib/pgbus/configuration.rb +21 -3
  71. data/lib/pgbus/generators/migration_detector.rb +40 -0
  72. data/lib/pgbus/instrumentation.rb +32 -0
  73. data/lib/pgbus/integrations/appsignal/dashboard.json +128 -1
  74. data/lib/pgbus/integrations/appsignal/event_span.rb +43 -0
  75. data/lib/pgbus/integrations/appsignal/job_span.rb +47 -0
  76. data/lib/pgbus/integrations/appsignal/span_subscriber.rb +128 -0
  77. data/lib/pgbus/integrations/appsignal/subscriber.rb +50 -100
  78. data/lib/pgbus/mcp/rack_app.rb +7 -2
  79. data/lib/pgbus/mcp/runner.rb +2 -1
  80. data/lib/pgbus/mcp/serialization.rb +25 -0
  81. data/lib/pgbus/mcp/server.rb +53 -10
  82. data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
  83. data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
  84. data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
  85. data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
  86. data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
  87. data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
  88. data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
  89. data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
  90. data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
  91. data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
  92. data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
  93. data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
  94. data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
  95. data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
  96. data/lib/pgbus/mcp/write_tool.rb +134 -0
  97. data/lib/pgbus/mcp.rb +15 -0
  98. data/lib/pgbus/metrics/backends/statsd.rb +6 -0
  99. data/lib/pgbus/metrics/subscriber.rb +14 -1
  100. data/lib/pgbus/metrics/workflow_metrics.rb +65 -0
  101. data/lib/pgbus/process/dispatcher.rb +26 -0
  102. data/lib/pgbus/version.rb +1 -1
  103. data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
  104. data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
  105. data/lib/pgbus/web/data_source/workflow_list.rb +248 -0
  106. data/lib/pgbus/web/data_source.rb +3 -0
  107. data/lib/pgbus/web/workflow_action_plan.rb +62 -0
  108. data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
  109. data/lib/pgbus/web/workflow_insights_report.rb +171 -0
  110. data/lib/pgbus/web/workflow_payload.rb +36 -0
  111. data/lib/pgbus/web/workflow_query.rb +82 -0
  112. data/lib/pgbus/web/workflow_state.rb +76 -0
  113. data/lib/pgbus/web/workflow_timeline.rb +113 -0
  114. data/lib/pgbus/web/workflow_window.rb +51 -0
  115. data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
  116. data/lib/pgbus/workflow/dsl.rb +105 -0
  117. data/lib/pgbus/workflow/error_capture.rb +63 -0
  118. data/lib/pgbus/workflow/exception_policy.rb +170 -0
  119. data/lib/pgbus/workflow/fan_out.rb +157 -0
  120. data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
  121. data/lib/pgbus/workflow/flow_control.rb +94 -0
  122. data/lib/pgbus/workflow/instance.rb +183 -0
  123. data/lib/pgbus/workflow/outside_verbs.rb +138 -0
  124. data/lib/pgbus/workflow/payload.rb +53 -0
  125. data/lib/pgbus/workflow/registry.rb +53 -0
  126. data/lib/pgbus/workflow/removed_step.rb +14 -0
  127. data/lib/pgbus/workflow/runner.rb +218 -0
  128. data/lib/pgbus/workflow/signal_matcher.rb +53 -0
  129. data/lib/pgbus/workflow/signals.rb +132 -0
  130. data/lib/pgbus/workflow/states.rb +119 -0
  131. data/lib/pgbus/workflow/step_context.rb +58 -0
  132. data/lib/pgbus/workflow/step_definition.rb +131 -0
  133. data/lib/pgbus/workflow/step_execution.rb +242 -0
  134. data/lib/pgbus/workflow/step_job.rb +28 -0
  135. data/lib/pgbus/workflow/step_outcome.rb +151 -0
  136. data/lib/pgbus/workflow/steps.rb +103 -0
  137. data/lib/pgbus/workflow/sweep.rb +156 -0
  138. data/lib/pgbus/workflow/test_helpers.rb +152 -0
  139. data/lib/pgbus/workflow/wait_timeout.rb +73 -0
  140. data/lib/pgbus/workflow/waiting.rb +82 -0
  141. data/lib/pgbus/workflow.rb +183 -0
  142. data/sig/pgbus/configuration.rbs +22 -0
  143. metadata +95 -1
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ class Configuration
5
+ # Workflows (issue #540): durable multi-step Pgbus::Workflow runs.
6
+ #
7
+ # config.workflow_queue = "workflows" # where StepJob messages go
8
+ # config.workflow_retention = 30.days # finished/cancelled runs kept this long; nil keeps them
9
+ # config.workflow_sweep_interval = 5.minutes # how often the dispatcher repairs stuck steps
10
+ # config.workflow_stall_threshold = 1.hour # an in_progress step older than this is stalled
11
+ # config.workflow_stall_action = :pause # :pause, :reattempt or :cancel a stalled step
12
+ # config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
13
+ # config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
14
+ # config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
15
+ # # wait_timeout: counts as stuck (waiting_overdue)
16
+ # config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
17
+ # # the dashboard's subject link (nil: plain text)
18
+ module WorkflowSettings
19
+ WORKFLOW_STALL_ACTIONS = %i[pause reattempt cancel].freeze
20
+
21
+ attr_reader :workflow_queue, :workflow_retention, :workflow_sweep_interval, :workflow_stall_threshold,
22
+ :workflow_stall_action, :workflow_waiting_visibility_threshold
23
+ attr_accessor :workflow_max_payload_size, :workflow_error_backtrace_lines, :workflow_subject_url
24
+
25
+ def workflow_queue=(value)
26
+ name = value.to_s
27
+ raise Pgbus::ConfigurationError, "workflow_queue must be a non-empty queue name" if value.nil? || name.strip.empty?
28
+
29
+ @workflow_queue = name
30
+ end
31
+
32
+ def workflow_retention=(value)
33
+ @workflow_retention = coerce_duration!(value, :workflow_retention)
34
+ end
35
+
36
+ def workflow_sweep_interval=(value)
37
+ @workflow_sweep_interval = coerce_duration!(value, :workflow_sweep_interval)
38
+ end
39
+
40
+ def workflow_stall_threshold=(value)
41
+ @workflow_stall_threshold = coerce_duration!(value, :workflow_stall_threshold)
42
+ end
43
+
44
+ def workflow_waiting_visibility_threshold=(value)
45
+ @workflow_waiting_visibility_threshold = coerce_duration!(value, :workflow_waiting_visibility_threshold)
46
+ end
47
+
48
+ def workflow_stall_action=(value)
49
+ action = value.respond_to?(:to_sym) ? value.to_sym : value
50
+ unless WORKFLOW_STALL_ACTIONS.include?(action)
51
+ raise Pgbus::ConfigurationError,
52
+ "workflow_stall_action must be one of #{WORKFLOW_STALL_ACTIONS.map(&:inspect).join(", ")}, got #{value.inspect}"
53
+ end
54
+
55
+ @workflow_stall_action = action
56
+ end
57
+
58
+ private
59
+
60
+ def initialize_workflow_settings
61
+ @workflow_queue = "default"
62
+ @workflow_retention = 30 * 24 * 3600 # 30 days
63
+ @workflow_sweep_interval = 300 # 5 minutes
64
+ @workflow_stall_threshold = 3600 # 1 hour
65
+ @workflow_stall_action = :pause
66
+ @workflow_max_payload_size = 131_072 # 128 KB
67
+ @workflow_error_backtrace_lines = 50
68
+ @workflow_waiting_visibility_threshold = 7 * 24 * 3600 # 7 days
69
+ @workflow_subject_url = nil
70
+ end
71
+
72
+ def validate_workflow_settings!
73
+ { workflow_sweep_interval: workflow_sweep_interval, workflow_stall_threshold: workflow_stall_threshold,
74
+ workflow_waiting_visibility_threshold: workflow_waiting_visibility_threshold,
75
+ workflow_retention: workflow_retention }.each do |name, value|
76
+ next if value.nil? && name == :workflow_retention
77
+ next if value.is_a?(Numeric) && value.positive? && value.finite?
78
+
79
+ raise Pgbus::ConfigurationError, "#{name} must be a finite number of seconds > 0"
80
+ end
81
+
82
+ begin
83
+ QueueNameValidator.validate!(queue_name(workflow_queue))
84
+ rescue ArgumentError => e
85
+ raise Pgbus::ConfigurationError, "workflow_queue #{workflow_queue.inspect}: #{e.message}"
86
+ end
87
+
88
+ { workflow_max_payload_size: workflow_max_payload_size,
89
+ workflow_error_backtrace_lines: workflow_error_backtrace_lines }.each do |name, value|
90
+ raise Pgbus::ConfigurationError, "#{name} must be a positive integer" unless value.is_a?(Integer) && value.positive?
91
+ end
92
+
93
+ return if workflow_subject_url.nil? || workflow_subject_url.respond_to?(:call)
94
+
95
+ raise Pgbus::ConfigurationError, "workflow_subject_url must respond to #call (a Proc/lambda) or be nil"
96
+ end
97
+ end
98
+ end
99
+ end
@@ -7,6 +7,7 @@ module Pgbus
7
7
  include PresenceSettings
8
8
  include StreamAuthorization
9
9
  include DeploySettings
10
+ include WorkflowSettings
10
11
 
11
12
  # Connection settings
12
13
  attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
@@ -215,10 +216,16 @@ module Pgbus
215
216
 
216
217
  # Web dashboard
217
218
  attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
218
- :insights_default_minutes, :base_controller_class, :return_to_app_url,
219
+ :insights_default_minutes, :base_controller_class, :return_to_app_url, :web_actor,
219
220
  :metrics_enabled,
220
221
  :web_filter_parameters, :web_filter_sensitive
221
222
 
223
+ # The MCP server's workflow write tools (issue #550): pgbus_workflow_resume,
224
+ # _cancel, _skip_step, _resume_at, _run_now and _signal. Off by default, and
225
+ # then they are never built, so tools/list cannot show them. Read once, when
226
+ # Pgbus::MCP::Server.build runs (pgbus mcp, Pgbus::MCP.rack_app).
227
+ attr_accessor :mcp_write_tools
228
+
222
229
  # HTTP health endpoints (liveness/readiness for orchestrators like
223
230
  # Kubernetes). `health_port` nil (default) leaves the standalone server in
224
231
  # the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
@@ -365,6 +372,7 @@ module Pgbus
365
372
  @recurring_enabled = true
366
373
  @recurring_execution_retention = 7 * 24 * 3600 # 7 days
367
374
  initialize_deploy_settings
375
+ initialize_workflow_settings
368
376
 
369
377
  @zombie_detection = true
370
378
 
@@ -380,7 +388,13 @@ module Pgbus
380
388
  @require_primary = false
381
389
  @connection_guc_mode = :options
382
390
 
391
+ @mcp_write_tools = false
392
+
383
393
  @web_auth = nil
394
+ # ->(request) { String or nil }: who an operator action on the dashboard
395
+ # is recorded as ("dashboard:<name>"; nil → "dashboard"). web_auth only
396
+ # answers allow/deny, so the name comes from here.
397
+ @web_actor = nil
384
398
  @web_refresh_interval = 5000
385
399
  @web_per_page = 25
386
400
  @web_live_updates = true
@@ -941,12 +955,14 @@ module Pgbus
941
955
  end
942
956
 
943
957
  raise Pgbus::ConfigurationError, "require_primary must be true or false" unless [true, false].include?(require_primary)
958
+ raise Pgbus::ConfigurationError, "mcp_write_tools must be true or false" unless [true, false].include?(mcp_write_tools)
944
959
 
945
960
  validate_job_path_gaps!
946
961
  validate_streams!
947
962
  validate_metrics_backend!
948
963
  validate_fair_share!
949
964
  validate_deploy_settings!
965
+ validate_workflow_settings!
950
966
 
951
967
  self
952
968
  end
@@ -1062,8 +1078,10 @@ module Pgbus
1062
1078
 
1063
1079
  # web_auth gates the dashboard; a non-callable value silently leaves it
1064
1080
  # open (mirror streams_presence_member).
1065
- unless web_auth.nil? || web_auth.respond_to?(:call)
1066
- raise Pgbus::ConfigurationError, "web_auth must respond to #call (a Proc/lambda) or be nil"
1081
+ { web_auth: web_auth, web_actor: web_actor }.each do |name, value|
1082
+ next if value.nil? || value.respond_to?(:call)
1083
+
1084
+ raise Pgbus::ConfigurationError, "#{name} must respond to #call (a Proc/lambda) or be nil"
1067
1085
  end
1068
1086
 
1069
1087
  raise Pgbus::ConfigurationError, "error_reporters must be an Array" unless error_reporters.is_a?(Array)
@@ -57,6 +57,22 @@ module Pgbus
57
57
  pgbus_batches
58
58
  ].freeze
59
59
 
60
+ # The three tables pgbus:add_workflows creates (issue #540).
61
+ WORKFLOW_TABLES = %w[pgbus_workflows pgbus_workflow_steps pgbus_workflow_events].freeze
62
+
63
+ # The waiting-step columns pgbus:add_workflow_signals adds to
64
+ # pgbus_workflow_steps (issue #541), with the pgbus_workflow_signals table.
65
+ WORKFLOW_WAITING_COLUMNS = %w[batch_id wait_for signal_id waiting_since timeout_at].freeze
66
+ WORKFLOW_WAITING_INDEXES = Ractor.make_shareable(
67
+ [
68
+ %w[pgbus_workflow_steps idx_pgbus_workflow_steps_batch],
69
+ %w[pgbus_workflow_steps idx_pgbus_workflow_steps_timeout],
70
+ %w[pgbus_workflow_signals idx_pgbus_workflow_signals_idempotency],
71
+ %w[pgbus_workflow_signals idx_pgbus_workflow_signals_pending],
72
+ %w[pgbus_workflow_signals idx_pgbus_workflow_signals_claimed]
73
+ ]
74
+ )
75
+
60
76
  # generator_key → Rails generator name. Passed to Thor's invoke.
61
77
  #
62
78
  # Note: uniqueness_keys invokes add_uniqueness_keys, which only
@@ -78,6 +94,8 @@ module Pgbus
78
94
  add_outbox: "pgbus:add_outbox",
79
95
  add_recurring: "pgbus:add_recurring",
80
96
  add_deploy_tasks: "pgbus:add_deploy_tasks",
97
+ add_workflows: "pgbus:add_workflows",
98
+ add_workflow_signals: "pgbus:add_workflow_signals",
81
99
  add_failed_events_index: "pgbus:add_failed_events_index",
82
100
  add_processed_event_completion: "pgbus:add_processed_event_completion",
83
101
  add_batch_executions: "pgbus:add_batch_executions",
@@ -101,6 +119,8 @@ module Pgbus
101
119
  add_outbox: "outbox entries table (transactional outbox)",
102
120
  add_recurring: "recurring tasks + executions tables",
103
121
  add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
122
+ add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
123
+ add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
104
124
  add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
105
125
  add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
106
126
  add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
@@ -130,6 +150,7 @@ module Pgbus
130
150
  *outbox_migrations,
131
151
  *recurring_migrations,
132
152
  *deploy_tasks_migrations,
153
+ *workflows_migrations,
133
154
  *failed_events_index_migrations,
134
155
  *processed_event_completion_migrations,
135
156
  *batch_executions_migrations,
@@ -226,6 +247,25 @@ module Pgbus
226
247
  [:add_deploy_tasks]
227
248
  end
228
249
 
250
+ # The add_workflows template uses if_not_exists throughout, so a
251
+ # half-present set of the three tables is safe to queue again.
252
+ #
253
+ # add_workflows carries the waiting-step schema too, so a database with
254
+ # no steps table needs only that one; a steps table without the columns
255
+ # (or a missing signals table) predates waiting steps.
256
+ def workflows_migrations
257
+ migrations = []
258
+ migrations << :add_workflows unless WORKFLOW_TABLES.all? { |table| table_exists?(table) }
259
+ return migrations unless table_exists?("pgbus_workflow_steps")
260
+
261
+ columns = column_names("pgbus_workflow_steps")
262
+ waiting = WORKFLOW_WAITING_COLUMNS.all? { |column| columns.include?(column) } &&
263
+ table_exists?("pgbus_workflow_signals") &&
264
+ WORKFLOW_WAITING_INDEXES.all? { |table, index| index_exists?(table, index) }
265
+ migrations << :add_workflow_signals unless waiting
266
+ migrations
267
+ end
268
+
229
269
  def failed_events_index_migrations
230
270
  # pgbus_failed_events is created by pgbus:install, but the
231
271
  # unique (queue_name, msg_id) index was added later via its own
@@ -54,6 +54,38 @@ module Pgbus
54
54
  # pgbus.batch_sweep — dispatcher stalled-batch sweep
55
55
  # payload: stalled_for, stale_executions, orphan_rows,
56
56
  # started_batches, finished_batches
57
+ # pgbus.workflow.step — a workflow step attempt ended (body, guard, stall)
58
+ # payload: workflow_class, workflow_id, step_name, attempt,
59
+ # outcome (completed | reattempted | skipped | paused |
60
+ # cancelled | finished_early), status (the step row's),
61
+ # error_class (nil without an error), duration (seconds
62
+ # the body ran; 0.0 for a guard; nil for a stall),
63
+ # pickup_latency (seconds from scheduled_at to the claim)
64
+ # pgbus.workflow_finished — a workflow run finished
65
+ # payload: workflow_class, workflow_id
66
+ # pgbus.workflow_paused — a workflow run paused (an exception, a stall, pause!)
67
+ # payload: workflow_class, workflow_id, step_name,
68
+ # error_class (nil for a manual pause)
69
+ # pgbus.workflow_cancelled — a workflow run was cancelled
70
+ # payload: workflow_class, workflow_id, step_name
71
+ # pgbus.workflow.stall_detected — a step was stalled on the redelivery of the message that
72
+ # claimed it (detection: "redelivery": its worker died, or
73
+ # its heartbeat lapsed) or after it outlived its stall_at
74
+ # (detection: "sweep")
75
+ # payload: workflow_class, workflow_id, step_name, attempt,
76
+ # detection
77
+ # pgbus.workflow.signal — signal! stored a signal on a run (or found it stored:
78
+ # a redelivery with the same idempotency key)
79
+ # payload: workflow_class, workflow_id, name, duplicate,
80
+ # woke (true when it woke the step waiting for it)
81
+ # pgbus.workflow.wait_timed_out — a waiting step's wait_timeout: passed; its on_timeout
82
+ # applied (the sweep, or TestHelpers#expire_wait)
83
+ # payload: workflow_class, workflow_id, step_name, attempt,
84
+ # action (pause! | cancel! | skip! | run)
85
+ # pgbus.workflow.gauge — emitted by each dispatcher workflow sweep
86
+ # payload: statuses ([{ workflow_class:, status:, count: }]),
87
+ # stuck ([{ workflow_class:, reason:, count: }]), zero-filled
88
+ # across every known class; reasons include waiting_overdue
57
89
  #
58
90
  module Instrumentation
59
91
  module_function
@@ -4,7 +4,7 @@
4
4
  ],
5
5
  "dashboard": {
6
6
  "title": "Pgbus",
7
- "description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\n",
7
+ "description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\nWorkflow step outcomes and run times, and stuck workflows.\n",
8
8
  "visuals": [
9
9
  {
10
10
  "type": "timeseries",
@@ -393,6 +393,133 @@
393
393
  "tags": []
394
394
  }
395
395
  ]
396
+ },
397
+ {
398
+ "type": "timeseries",
399
+ "display": "LINE",
400
+ "title": "Workflow steps per outcome",
401
+ "description": "Workflow step attempts by class, step and outcome (completed, reattempted, skipped, paused, cancelled, finished_early).",
402
+ "line_label": "%workflow_class% - %step% - %outcome%",
403
+ "format": "number",
404
+ "draw_null_as_zero": true,
405
+ "metrics": [
406
+ {
407
+ "name": "pgbus_workflow_steps_total",
408
+ "fields": [
409
+ {
410
+ "field": "COUNTER"
411
+ }
412
+ ],
413
+ "tags": [
414
+ {
415
+ "key": "workflow_class",
416
+ "value": "*"
417
+ },
418
+ {
419
+ "key": "step",
420
+ "value": "*"
421
+ },
422
+ {
423
+ "key": "outcome",
424
+ "value": "*"
425
+ }
426
+ ]
427
+ }
428
+ ]
429
+ },
430
+ {
431
+ "type": "timeseries",
432
+ "display": "LINE",
433
+ "title": "Workflow step run time (seconds)",
434
+ "description": "Mean and p95 run time of each workflow step's body, in seconds.",
435
+ "line_label": "%workflow_class% - %step% - %outcome%",
436
+ "format": "number",
437
+ "draw_null_as_zero": true,
438
+ "metrics": [
439
+ {
440
+ "name": "pgbus_workflow_step_duration_seconds",
441
+ "fields": [
442
+ {
443
+ "field": "MEAN"
444
+ },
445
+ {
446
+ "field": "P95"
447
+ }
448
+ ],
449
+ "tags": [
450
+ {
451
+ "key": "workflow_class",
452
+ "value": "*"
453
+ },
454
+ {
455
+ "key": "step",
456
+ "value": "*"
457
+ },
458
+ {
459
+ "key": "outcome",
460
+ "value": "*"
461
+ }
462
+ ]
463
+ }
464
+ ]
465
+ },
466
+ {
467
+ "type": "timeseries",
468
+ "display": "LINE",
469
+ "title": "Workflows by status",
470
+ "description": "Workflow runs per class and status, from each dispatcher sweep.",
471
+ "line_label": "%workflow_class% - %status%",
472
+ "format": "number",
473
+ "draw_null_as_zero": true,
474
+ "metrics": [
475
+ {
476
+ "name": "pgbus_workflows",
477
+ "fields": [
478
+ {
479
+ "field": "GAUGE"
480
+ }
481
+ ],
482
+ "tags": [
483
+ {
484
+ "key": "workflow_class",
485
+ "value": "*"
486
+ },
487
+ {
488
+ "key": "status",
489
+ "value": "*"
490
+ }
491
+ ]
492
+ }
493
+ ]
494
+ },
495
+ {
496
+ "type": "timeseries",
497
+ "display": "LINE",
498
+ "title": "Stuck workflows",
499
+ "description": "Ongoing workflow runs that need a person, by class and reason (paused_error, stalled, paused_manual, overdue).",
500
+ "line_label": "%workflow_class% - %reason%",
501
+ "format": "number",
502
+ "draw_null_as_zero": true,
503
+ "metrics": [
504
+ {
505
+ "name": "pgbus_workflows_stuck",
506
+ "fields": [
507
+ {
508
+ "field": "GAUGE"
509
+ }
510
+ ],
511
+ "tags": [
512
+ {
513
+ "key": "workflow_class",
514
+ "value": "*"
515
+ },
516
+ {
517
+ "key": "reason",
518
+ "value": "*"
519
+ }
520
+ ]
521
+ }
522
+ ]
396
523
  }
397
524
  ]
398
525
  }
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "pgbus/integrations/appsignal/span_subscriber"
4
+
5
+ module Pgbus
6
+ module Integrations
7
+ module Appsignal
8
+ # The `<Handler>#handle` transaction around pgbus.event_processed, and
9
+ # the fallback for an error pgbus.event_failed reports from outside it
10
+ # (an event that could not be built never reaches the span).
11
+ class EventSpan < SpanSubscriber
12
+ private
13
+
14
+ def frame_key = :pgbus_appsignal_event_span
15
+
16
+ def attached_key = :pgbus_appsignal_event_span_error
17
+
18
+ def describe(transaction, payload)
19
+ transaction.set_action_if_nil("#{payload[:handler] || "UnknownHandler"}#handle")
20
+ apply_queue_start(transaction, payload[:published_at])
21
+ transaction.add_tags(handler_tags(payload))
22
+ end
23
+
24
+ # The processed count is taken at finish whether or not the handler
25
+ # raised, as Metrics::Subscriber does: a raised handler counts as
26
+ # processed and, through pgbus.event_failed, as failed.
27
+ def finished(payload, duration)
28
+ tags = { handler: payload[:handler], routing_key: payload[:routing_key] }
29
+ ::Appsignal.add_distribution_value("pgbus_event_duration_ms", duration, tags)
30
+ ::Appsignal.increment_counter("pgbus_event_count", 1, tags.merge(status: "processed"))
31
+ end
32
+
33
+ def handler_tags(payload)
34
+ {
35
+ "handler" => payload[:handler],
36
+ "routing_key" => payload[:routing_key],
37
+ "attempts" => payload[:read_ct]
38
+ }.compact
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "pgbus/integrations/appsignal/span_subscriber"
4
+
5
+ module Pgbus
6
+ module Integrations
7
+ module Appsignal
8
+ # The `<JobClass>#perform` transaction around pgbus.executor.execute
9
+ # (deserialize + perform + archive), and the fallback for an error
10
+ # pgbus.job_failed reports from outside it.
11
+ class JobSpan < SpanSubscriber
12
+ private
13
+
14
+ def frame_key = :pgbus_appsignal_job_span
15
+
16
+ def attached_key = :pgbus_appsignal_job_span_error
17
+
18
+ def describe(transaction, payload)
19
+ transaction.set_action_if_nil("#{payload[:job_class] || "UnknownJob"}#perform")
20
+ apply_queue_start(transaction, payload[:enqueued_at])
21
+ transaction.add_tags(job_tags(payload))
22
+ transaction.add_params_if_nil { { arguments: payload[:arguments] } } if payload.key?(:arguments)
23
+ end
24
+
25
+ def finished(payload, duration)
26
+ ::Appsignal.add_distribution_value(
27
+ "pgbus_job_duration_ms",
28
+ duration,
29
+ { queue: payload[:queue], job_class: payload[:job_class] }
30
+ )
31
+ end
32
+
33
+ def job_tags(payload)
34
+ tags = {
35
+ "queue" => payload[:queue],
36
+ "job_class" => payload[:job_class],
37
+ "attempts" => payload[:read_ct]
38
+ }
39
+ tags["active_job_id"] = payload[:job_id] if payload[:job_id]
40
+ tags["provider_job_id"] = payload[:provider_job_id] if payload[:provider_job_id]
41
+ tags["request_id"] = payload[:provider_job_id] || payload[:job_id]
42
+ tags.compact
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,128 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Pgbus
6
+ module Integrations
7
+ module Appsignal
8
+ # The AppSignal transaction of one unit of work: a job (JobSpan, on
9
+ # pgbus.executor.execute) or an event handler (EventSpan, on
10
+ # pgbus.event_processed).
11
+ #
12
+ # An ActiveSupport::Notifications listener that responds to #start and
13
+ # #finish is called on both sides of the instrumented block (the Evented
14
+ # subscriber). A finish-only block subscriber runs after the block has
15
+ # returned, so the transaction it opened was never current while the job
16
+ # ran: job code had nothing to attach to, and pgbus.job_failed, fired
17
+ # later, found no transaction to put its error on (issue #552). Here
18
+ # #start opens the transaction before the block runs, so job and handler
19
+ # code, and the Rails error reporter, find it current; #finish records
20
+ # the exception the block raised (the instrumenter puts it in
21
+ # payload[:exception_object]) and completes the transaction.
22
+ #
23
+ # Like AppSignal's own ActiveJob hook, a transaction that is already
24
+ # current is reused and left to its owner to complete.
25
+ #
26
+ # The frame lives in Thread.current[], which is fiber-local, so start and
27
+ # finish pair up per job under execution_mode: :async, the same way
28
+ # AppSignal keeps its current transaction. A span opened inside another
29
+ # of its kind (a handler calling another handler's #process) keeps the
30
+ # enclosing frame and restores it at finish, so the outer span still
31
+ # completes its transaction. An instance holds no state of its own.
32
+ # Nothing an AppSignal call raises reaches the job: it is logged at warn
33
+ # level.
34
+ #
35
+ # Subclasses define #frame_key and #attached_key (the fiber-local slots),
36
+ # #describe(transaction, payload) (action, queue start, tags) and
37
+ # #finished(payload, duration_ms) (the span's metrics).
38
+ class SpanSubscriber
39
+ BACKGROUND_JOB = "background_job"
40
+
41
+ # owned: this span created the transaction, so it completes it.
42
+ # previous: the enclosing span's frame, restored at finish.
43
+ Frame = Struct.new(:started_at, :owned, :previous)
44
+
45
+ def start(_name, _id, payload)
46
+ frame = Thread.current[frame_key] = Frame.new(monotonic_now, false, Thread.current[frame_key])
47
+ guard do
48
+ frame.owned = !::Appsignal::Transaction.current?
49
+ transaction = frame.owned ? ::Appsignal::Transaction.create(BACKGROUND_JOB) : ::Appsignal::Transaction.current
50
+ describe(transaction, payload)
51
+ end
52
+ end
53
+
54
+ def finish(_name, _id, payload)
55
+ frame = Thread.current[frame_key]
56
+ guard do
57
+ Thread.current[attached_key] = attach(payload[:exception_object])
58
+ finished(payload, monotonic_now - frame.started_at)
59
+ end
60
+ ensure
61
+ guard do
62
+ Thread.current[frame_key] = frame.previous
63
+ ::Appsignal::Transaction.complete_current! if frame.owned
64
+ end
65
+ end
66
+
67
+ # pgbus.job_failed and pgbus.event_failed fire after #finish. An error
68
+ # the span recorded is already on its transaction. One raised before
69
+ # the span opened (a message body that is not JSON, an event that
70
+ # cannot be built, a dead-letter move that failed) had no transaction,
71
+ # so it is reported on one of its own, named and tagged like the span's
72
+ # (or added to a transaction that is current, which is left as it is).
73
+ def report_unattached(error, payload)
74
+ attached = Thread.current[attached_key]
75
+ Thread.current[attached_key] = nil
76
+ return if error.nil? || error.equal?(attached)
77
+ return ::Appsignal::Transaction.current.set_error(error) if ::Appsignal::Transaction.current?
78
+
79
+ ::Appsignal.report_error(error) do |transaction|
80
+ transaction.set_namespace(BACKGROUND_JOB)
81
+ describe(transaction, payload)
82
+ end
83
+ end
84
+
85
+ private
86
+
87
+ # Returns the exception it recorded, nil when there was none.
88
+ def attach(error)
89
+ return unless error && ::Appsignal::Transaction.current?
90
+
91
+ ::Appsignal::Transaction.current.set_error(error)
92
+ error
93
+ end
94
+
95
+ # AppSignal expects queue-start as Unix epoch milliseconds. Pgbus
96
+ # carries it as either an ISO-8601 String or a Time — both happen
97
+ # in practice (executor passes the JSON string, handler passes a
98
+ # parsed Time).
99
+ def apply_queue_start(transaction, value)
100
+ return unless value
101
+
102
+ millis =
103
+ case value
104
+ when Time
105
+ (value.to_f * 1_000).to_i
106
+ when String
107
+ (Time.parse(value).to_f * 1_000).to_i
108
+ when Numeric
109
+ value.to_i
110
+ end
111
+ transaction.set_queue_start(millis) if millis
112
+ rescue ArgumentError
113
+ # Unparseable timestamp — skip rather than blow up.
114
+ end
115
+
116
+ def guard
117
+ yield
118
+ rescue StandardError => e
119
+ Pgbus.logger.warn { "[Pgbus::AppSignal] subscriber error: #{e.class}: #{e.message}" }
120
+ end
121
+
122
+ def monotonic_now
123
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :float_millisecond)
124
+ end
125
+ end
126
+ end
127
+ end
128
+ end