pgbus 0.19.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +104 -9
- data/Rakefile +6 -1
- data/app/controllers/pgbus/dashboard_controller.rb +2 -0
- data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
- data/app/controllers/pgbus/workflows_controller.rb +197 -0
- data/app/frontend/pgbus/modules/charts.js +130 -4
- data/app/frontend/pgbus/modules/charts.min.js +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/frontend/pgbus/style.css +1 -1
- data/app/helpers/pgbus/button_helper.rb +4 -2
- data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
- data/app/helpers/pgbus/workflows_helper.rb +165 -0
- data/app/models/pgbus/workflow_entry.rb +172 -0
- data/app/models/pgbus/workflow_event.rb +43 -0
- data/app/models/pgbus/workflow_signal.rb +77 -0
- data/app/models/pgbus/workflow_step.rb +142 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
- data/app/views/pgbus/workflows/_actions.html.erb +59 -0
- data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
- data/app/views/pgbus/workflows/_filters.html.erb +29 -0
- data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
- data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
- data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
- data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
- data/app/views/pgbus/workflows/_timeline.html.erb +39 -0
- data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
- data/app/views/pgbus/workflows/attention.html.erb +88 -0
- data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
- data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
- data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
- data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
- data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
- data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
- data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
- data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
- data/app/views/pgbus/workflows/index.html.erb +15 -0
- data/app/views/pgbus/workflows/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/show.html.erb +93 -0
- data/config/i18n-tasks.yml +6 -0
- data/config/locales/da.yml +442 -0
- data/config/locales/de.yml +442 -0
- data/config/locales/en.yml +442 -0
- data/config/locales/es.yml +442 -0
- data/config/locales/fi.yml +442 -0
- data/config/locales/fr.yml +442 -0
- data/config/locales/it.yml +442 -0
- data/config/locales/ja.yml +442 -0
- data/config/locales/nb.yml +442 -0
- data/config/locales/nl.yml +442 -0
- data/config/locales/pt.yml +442 -0
- data/config/locales/sv.yml +442 -0
- data/config/routes.rb +26 -0
- data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
- data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
- data/lib/pgbus/active_job/adapter.rb +67 -26
- data/lib/pgbus/active_job/batch_id.rb +23 -1
- data/lib/pgbus/active_job/delivery.rb +49 -0
- data/lib/pgbus/active_job/executor.rb +3 -1
- data/lib/pgbus/batch/builder.rb +1 -0
- data/lib/pgbus/batch.rb +35 -3
- data/lib/pgbus/configuration/workflow_settings.rb +99 -0
- data/lib/pgbus/configuration.rb +21 -3
- data/lib/pgbus/generators/migration_detector.rb +40 -0
- data/lib/pgbus/instrumentation.rb +32 -0
- data/lib/pgbus/integrations/appsignal/dashboard.json +128 -1
- data/lib/pgbus/integrations/appsignal/event_span.rb +43 -0
- data/lib/pgbus/integrations/appsignal/job_span.rb +47 -0
- data/lib/pgbus/integrations/appsignal/span_subscriber.rb +128 -0
- data/lib/pgbus/integrations/appsignal/subscriber.rb +50 -100
- data/lib/pgbus/mcp/rack_app.rb +7 -2
- data/lib/pgbus/mcp/runner.rb +2 -1
- data/lib/pgbus/mcp/serialization.rb +25 -0
- data/lib/pgbus/mcp/server.rb +53 -10
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
- data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
- data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
- data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
- data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
- data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
- data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
- data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
- data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
- data/lib/pgbus/mcp/write_tool.rb +134 -0
- data/lib/pgbus/mcp.rb +15 -0
- data/lib/pgbus/metrics/backends/statsd.rb +6 -0
- data/lib/pgbus/metrics/subscriber.rb +14 -1
- data/lib/pgbus/metrics/workflow_metrics.rb +65 -0
- data/lib/pgbus/process/dispatcher.rb +26 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
- data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
- data/lib/pgbus/web/data_source/workflow_list.rb +248 -0
- data/lib/pgbus/web/data_source.rb +3 -0
- data/lib/pgbus/web/workflow_action_plan.rb +62 -0
- data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
- data/lib/pgbus/web/workflow_insights_report.rb +171 -0
- data/lib/pgbus/web/workflow_payload.rb +36 -0
- data/lib/pgbus/web/workflow_query.rb +82 -0
- data/lib/pgbus/web/workflow_state.rb +76 -0
- data/lib/pgbus/web/workflow_timeline.rb +113 -0
- data/lib/pgbus/web/workflow_window.rb +51 -0
- data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
- data/lib/pgbus/workflow/dsl.rb +105 -0
- data/lib/pgbus/workflow/error_capture.rb +63 -0
- data/lib/pgbus/workflow/exception_policy.rb +170 -0
- data/lib/pgbus/workflow/fan_out.rb +157 -0
- data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
- data/lib/pgbus/workflow/flow_control.rb +94 -0
- data/lib/pgbus/workflow/instance.rb +183 -0
- data/lib/pgbus/workflow/outside_verbs.rb +138 -0
- data/lib/pgbus/workflow/payload.rb +53 -0
- data/lib/pgbus/workflow/registry.rb +53 -0
- data/lib/pgbus/workflow/removed_step.rb +14 -0
- data/lib/pgbus/workflow/runner.rb +218 -0
- data/lib/pgbus/workflow/signal_matcher.rb +53 -0
- data/lib/pgbus/workflow/signals.rb +132 -0
- data/lib/pgbus/workflow/states.rb +119 -0
- data/lib/pgbus/workflow/step_context.rb +58 -0
- data/lib/pgbus/workflow/step_definition.rb +131 -0
- data/lib/pgbus/workflow/step_execution.rb +242 -0
- data/lib/pgbus/workflow/step_job.rb +28 -0
- data/lib/pgbus/workflow/step_outcome.rb +151 -0
- data/lib/pgbus/workflow/steps.rb +103 -0
- data/lib/pgbus/workflow/sweep.rb +156 -0
- data/lib/pgbus/workflow/test_helpers.rb +152 -0
- data/lib/pgbus/workflow/wait_timeout.rb +73 -0
- data/lib/pgbus/workflow/waiting.rb +82 -0
- data/lib/pgbus/workflow.rb +183 -0
- data/sig/pgbus/configuration.rbs +22 -0
- metadata +95 -1
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Configuration
|
|
5
|
+
# Workflows (issue #540): durable multi-step Pgbus::Workflow runs.
|
|
6
|
+
#
|
|
7
|
+
# config.workflow_queue = "workflows" # where StepJob messages go
|
|
8
|
+
# config.workflow_retention = 30.days # finished/cancelled runs kept this long; nil keeps them
|
|
9
|
+
# config.workflow_sweep_interval = 5.minutes # how often the dispatcher repairs stuck steps
|
|
10
|
+
# config.workflow_stall_threshold = 1.hour # an in_progress step older than this is stalled
|
|
11
|
+
# config.workflow_stall_action = :pause # :pause, :reattempt or :cancel a stalled step
|
|
12
|
+
# config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
|
|
13
|
+
# config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
|
|
14
|
+
# config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
|
|
15
|
+
# # wait_timeout: counts as stuck (waiting_overdue)
|
|
16
|
+
# config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
|
|
17
|
+
# # the dashboard's subject link (nil: plain text)
|
|
18
|
+
module WorkflowSettings
|
|
19
|
+
WORKFLOW_STALL_ACTIONS = %i[pause reattempt cancel].freeze
|
|
20
|
+
|
|
21
|
+
attr_reader :workflow_queue, :workflow_retention, :workflow_sweep_interval, :workflow_stall_threshold,
|
|
22
|
+
:workflow_stall_action, :workflow_waiting_visibility_threshold
|
|
23
|
+
attr_accessor :workflow_max_payload_size, :workflow_error_backtrace_lines, :workflow_subject_url
|
|
24
|
+
|
|
25
|
+
def workflow_queue=(value)
|
|
26
|
+
name = value.to_s
|
|
27
|
+
raise Pgbus::ConfigurationError, "workflow_queue must be a non-empty queue name" if value.nil? || name.strip.empty?
|
|
28
|
+
|
|
29
|
+
@workflow_queue = name
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def workflow_retention=(value)
|
|
33
|
+
@workflow_retention = coerce_duration!(value, :workflow_retention)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def workflow_sweep_interval=(value)
|
|
37
|
+
@workflow_sweep_interval = coerce_duration!(value, :workflow_sweep_interval)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def workflow_stall_threshold=(value)
|
|
41
|
+
@workflow_stall_threshold = coerce_duration!(value, :workflow_stall_threshold)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def workflow_waiting_visibility_threshold=(value)
|
|
45
|
+
@workflow_waiting_visibility_threshold = coerce_duration!(value, :workflow_waiting_visibility_threshold)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def workflow_stall_action=(value)
|
|
49
|
+
action = value.respond_to?(:to_sym) ? value.to_sym : value
|
|
50
|
+
unless WORKFLOW_STALL_ACTIONS.include?(action)
|
|
51
|
+
raise Pgbus::ConfigurationError,
|
|
52
|
+
"workflow_stall_action must be one of #{WORKFLOW_STALL_ACTIONS.map(&:inspect).join(", ")}, got #{value.inspect}"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
@workflow_stall_action = action
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def initialize_workflow_settings
|
|
61
|
+
@workflow_queue = "default"
|
|
62
|
+
@workflow_retention = 30 * 24 * 3600 # 30 days
|
|
63
|
+
@workflow_sweep_interval = 300 # 5 minutes
|
|
64
|
+
@workflow_stall_threshold = 3600 # 1 hour
|
|
65
|
+
@workflow_stall_action = :pause
|
|
66
|
+
@workflow_max_payload_size = 131_072 # 128 KB
|
|
67
|
+
@workflow_error_backtrace_lines = 50
|
|
68
|
+
@workflow_waiting_visibility_threshold = 7 * 24 * 3600 # 7 days
|
|
69
|
+
@workflow_subject_url = nil
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def validate_workflow_settings!
|
|
73
|
+
{ workflow_sweep_interval: workflow_sweep_interval, workflow_stall_threshold: workflow_stall_threshold,
|
|
74
|
+
workflow_waiting_visibility_threshold: workflow_waiting_visibility_threshold,
|
|
75
|
+
workflow_retention: workflow_retention }.each do |name, value|
|
|
76
|
+
next if value.nil? && name == :workflow_retention
|
|
77
|
+
next if value.is_a?(Numeric) && value.positive? && value.finite?
|
|
78
|
+
|
|
79
|
+
raise Pgbus::ConfigurationError, "#{name} must be a finite number of seconds > 0"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
begin
|
|
83
|
+
QueueNameValidator.validate!(queue_name(workflow_queue))
|
|
84
|
+
rescue ArgumentError => e
|
|
85
|
+
raise Pgbus::ConfigurationError, "workflow_queue #{workflow_queue.inspect}: #{e.message}"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
{ workflow_max_payload_size: workflow_max_payload_size,
|
|
89
|
+
workflow_error_backtrace_lines: workflow_error_backtrace_lines }.each do |name, value|
|
|
90
|
+
raise Pgbus::ConfigurationError, "#{name} must be a positive integer" unless value.is_a?(Integer) && value.positive?
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
return if workflow_subject_url.nil? || workflow_subject_url.respond_to?(:call)
|
|
94
|
+
|
|
95
|
+
raise Pgbus::ConfigurationError, "workflow_subject_url must respond to #call (a Proc/lambda) or be nil"
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -7,6 +7,7 @@ module Pgbus
|
|
|
7
7
|
include PresenceSettings
|
|
8
8
|
include StreamAuthorization
|
|
9
9
|
include DeploySettings
|
|
10
|
+
include WorkflowSettings
|
|
10
11
|
|
|
11
12
|
# Connection settings
|
|
12
13
|
attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
|
|
@@ -215,10 +216,16 @@ module Pgbus
|
|
|
215
216
|
|
|
216
217
|
# Web dashboard
|
|
217
218
|
attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
|
|
218
|
-
:insights_default_minutes, :base_controller_class, :return_to_app_url,
|
|
219
|
+
:insights_default_minutes, :base_controller_class, :return_to_app_url, :web_actor,
|
|
219
220
|
:metrics_enabled,
|
|
220
221
|
:web_filter_parameters, :web_filter_sensitive
|
|
221
222
|
|
|
223
|
+
# The MCP server's workflow write tools (issue #550): pgbus_workflow_resume,
|
|
224
|
+
# _cancel, _skip_step, _resume_at, _run_now and _signal. Off by default, and
|
|
225
|
+
# then they are never built, so tools/list cannot show them. Read once, when
|
|
226
|
+
# Pgbus::MCP::Server.build runs (pgbus mcp, Pgbus::MCP.rack_app).
|
|
227
|
+
attr_accessor :mcp_write_tools
|
|
228
|
+
|
|
222
229
|
# HTTP health endpoints (liveness/readiness for orchestrators like
|
|
223
230
|
# Kubernetes). `health_port` nil (default) leaves the standalone server in
|
|
224
231
|
# the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
|
|
@@ -365,6 +372,7 @@ module Pgbus
|
|
|
365
372
|
@recurring_enabled = true
|
|
366
373
|
@recurring_execution_retention = 7 * 24 * 3600 # 7 days
|
|
367
374
|
initialize_deploy_settings
|
|
375
|
+
initialize_workflow_settings
|
|
368
376
|
|
|
369
377
|
@zombie_detection = true
|
|
370
378
|
|
|
@@ -380,7 +388,13 @@ module Pgbus
|
|
|
380
388
|
@require_primary = false
|
|
381
389
|
@connection_guc_mode = :options
|
|
382
390
|
|
|
391
|
+
@mcp_write_tools = false
|
|
392
|
+
|
|
383
393
|
@web_auth = nil
|
|
394
|
+
# ->(request) { String or nil }: who an operator action on the dashboard
|
|
395
|
+
# is recorded as ("dashboard:<name>"; nil → "dashboard"). web_auth only
|
|
396
|
+
# answers allow/deny, so the name comes from here.
|
|
397
|
+
@web_actor = nil
|
|
384
398
|
@web_refresh_interval = 5000
|
|
385
399
|
@web_per_page = 25
|
|
386
400
|
@web_live_updates = true
|
|
@@ -941,12 +955,14 @@ module Pgbus
|
|
|
941
955
|
end
|
|
942
956
|
|
|
943
957
|
raise Pgbus::ConfigurationError, "require_primary must be true or false" unless [true, false].include?(require_primary)
|
|
958
|
+
raise Pgbus::ConfigurationError, "mcp_write_tools must be true or false" unless [true, false].include?(mcp_write_tools)
|
|
944
959
|
|
|
945
960
|
validate_job_path_gaps!
|
|
946
961
|
validate_streams!
|
|
947
962
|
validate_metrics_backend!
|
|
948
963
|
validate_fair_share!
|
|
949
964
|
validate_deploy_settings!
|
|
965
|
+
validate_workflow_settings!
|
|
950
966
|
|
|
951
967
|
self
|
|
952
968
|
end
|
|
@@ -1062,8 +1078,10 @@ module Pgbus
|
|
|
1062
1078
|
|
|
1063
1079
|
# web_auth gates the dashboard; a non-callable value silently leaves it
|
|
1064
1080
|
# open (mirror streams_presence_member).
|
|
1065
|
-
|
|
1066
|
-
|
|
1081
|
+
{ web_auth: web_auth, web_actor: web_actor }.each do |name, value|
|
|
1082
|
+
next if value.nil? || value.respond_to?(:call)
|
|
1083
|
+
|
|
1084
|
+
raise Pgbus::ConfigurationError, "#{name} must respond to #call (a Proc/lambda) or be nil"
|
|
1067
1085
|
end
|
|
1068
1086
|
|
|
1069
1087
|
raise Pgbus::ConfigurationError, "error_reporters must be an Array" unless error_reporters.is_a?(Array)
|
|
@@ -57,6 +57,22 @@ module Pgbus
|
|
|
57
57
|
pgbus_batches
|
|
58
58
|
].freeze
|
|
59
59
|
|
|
60
|
+
# The three tables pgbus:add_workflows creates (issue #540).
|
|
61
|
+
WORKFLOW_TABLES = %w[pgbus_workflows pgbus_workflow_steps pgbus_workflow_events].freeze
|
|
62
|
+
|
|
63
|
+
# The waiting-step columns pgbus:add_workflow_signals adds to
|
|
64
|
+
# pgbus_workflow_steps (issue #541), with the pgbus_workflow_signals table.
|
|
65
|
+
WORKFLOW_WAITING_COLUMNS = %w[batch_id wait_for signal_id waiting_since timeout_at].freeze
|
|
66
|
+
WORKFLOW_WAITING_INDEXES = Ractor.make_shareable(
|
|
67
|
+
[
|
|
68
|
+
%w[pgbus_workflow_steps idx_pgbus_workflow_steps_batch],
|
|
69
|
+
%w[pgbus_workflow_steps idx_pgbus_workflow_steps_timeout],
|
|
70
|
+
%w[pgbus_workflow_signals idx_pgbus_workflow_signals_idempotency],
|
|
71
|
+
%w[pgbus_workflow_signals idx_pgbus_workflow_signals_pending],
|
|
72
|
+
%w[pgbus_workflow_signals idx_pgbus_workflow_signals_claimed]
|
|
73
|
+
]
|
|
74
|
+
)
|
|
75
|
+
|
|
60
76
|
# generator_key → Rails generator name. Passed to Thor's invoke.
|
|
61
77
|
#
|
|
62
78
|
# Note: uniqueness_keys invokes add_uniqueness_keys, which only
|
|
@@ -78,6 +94,8 @@ module Pgbus
|
|
|
78
94
|
add_outbox: "pgbus:add_outbox",
|
|
79
95
|
add_recurring: "pgbus:add_recurring",
|
|
80
96
|
add_deploy_tasks: "pgbus:add_deploy_tasks",
|
|
97
|
+
add_workflows: "pgbus:add_workflows",
|
|
98
|
+
add_workflow_signals: "pgbus:add_workflow_signals",
|
|
81
99
|
add_failed_events_index: "pgbus:add_failed_events_index",
|
|
82
100
|
add_processed_event_completion: "pgbus:add_processed_event_completion",
|
|
83
101
|
add_batch_executions: "pgbus:add_batch_executions",
|
|
@@ -101,6 +119,8 @@ module Pgbus
|
|
|
101
119
|
add_outbox: "outbox entries table (transactional outbox)",
|
|
102
120
|
add_recurring: "recurring tasks + executions tables",
|
|
103
121
|
add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
|
|
122
|
+
add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
|
|
123
|
+
add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
|
|
104
124
|
add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
|
|
105
125
|
add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
|
|
106
126
|
add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
|
|
@@ -130,6 +150,7 @@ module Pgbus
|
|
|
130
150
|
*outbox_migrations,
|
|
131
151
|
*recurring_migrations,
|
|
132
152
|
*deploy_tasks_migrations,
|
|
153
|
+
*workflows_migrations,
|
|
133
154
|
*failed_events_index_migrations,
|
|
134
155
|
*processed_event_completion_migrations,
|
|
135
156
|
*batch_executions_migrations,
|
|
@@ -226,6 +247,25 @@ module Pgbus
|
|
|
226
247
|
[:add_deploy_tasks]
|
|
227
248
|
end
|
|
228
249
|
|
|
250
|
+
# The add_workflows template uses if_not_exists throughout, so a
|
|
251
|
+
# half-present set of the three tables is safe to queue again.
|
|
252
|
+
#
|
|
253
|
+
# add_workflows carries the waiting-step schema too, so a database with
|
|
254
|
+
# no steps table needs only that one; a steps table without the columns
|
|
255
|
+
# (or a missing signals table) predates waiting steps.
|
|
256
|
+
def workflows_migrations
|
|
257
|
+
migrations = []
|
|
258
|
+
migrations << :add_workflows unless WORKFLOW_TABLES.all? { |table| table_exists?(table) }
|
|
259
|
+
return migrations unless table_exists?("pgbus_workflow_steps")
|
|
260
|
+
|
|
261
|
+
columns = column_names("pgbus_workflow_steps")
|
|
262
|
+
waiting = WORKFLOW_WAITING_COLUMNS.all? { |column| columns.include?(column) } &&
|
|
263
|
+
table_exists?("pgbus_workflow_signals") &&
|
|
264
|
+
WORKFLOW_WAITING_INDEXES.all? { |table, index| index_exists?(table, index) }
|
|
265
|
+
migrations << :add_workflow_signals unless waiting
|
|
266
|
+
migrations
|
|
267
|
+
end
|
|
268
|
+
|
|
229
269
|
def failed_events_index_migrations
|
|
230
270
|
# pgbus_failed_events is created by pgbus:install, but the
|
|
231
271
|
# unique (queue_name, msg_id) index was added later via its own
|
|
@@ -54,6 +54,38 @@ module Pgbus
|
|
|
54
54
|
# pgbus.batch_sweep — dispatcher stalled-batch sweep
|
|
55
55
|
# payload: stalled_for, stale_executions, orphan_rows,
|
|
56
56
|
# started_batches, finished_batches
|
|
57
|
+
# pgbus.workflow.step — a workflow step attempt ended (body, guard, stall)
|
|
58
|
+
# payload: workflow_class, workflow_id, step_name, attempt,
|
|
59
|
+
# outcome (completed | reattempted | skipped | paused |
|
|
60
|
+
# cancelled | finished_early), status (the step row's),
|
|
61
|
+
# error_class (nil without an error), duration (seconds
|
|
62
|
+
# the body ran; 0.0 for a guard; nil for a stall),
|
|
63
|
+
# pickup_latency (seconds from scheduled_at to the claim)
|
|
64
|
+
# pgbus.workflow_finished — a workflow run finished
|
|
65
|
+
# payload: workflow_class, workflow_id
|
|
66
|
+
# pgbus.workflow_paused — a workflow run paused (an exception, a stall, pause!)
|
|
67
|
+
# payload: workflow_class, workflow_id, step_name,
|
|
68
|
+
# error_class (nil for a manual pause)
|
|
69
|
+
# pgbus.workflow_cancelled — a workflow run was cancelled
|
|
70
|
+
# payload: workflow_class, workflow_id, step_name
|
|
71
|
+
# pgbus.workflow.stall_detected — a step was stalled on the redelivery of the message that
|
|
72
|
+
# claimed it (detection: "redelivery": its worker died, or
|
|
73
|
+
# its heartbeat lapsed) or after it outlived its stall_at
|
|
74
|
+
# (detection: "sweep")
|
|
75
|
+
# payload: workflow_class, workflow_id, step_name, attempt,
|
|
76
|
+
# detection
|
|
77
|
+
# pgbus.workflow.signal — signal! stored a signal on a run (or found it stored:
|
|
78
|
+
# a redelivery with the same idempotency key)
|
|
79
|
+
# payload: workflow_class, workflow_id, name, duplicate,
|
|
80
|
+
# woke (true when it woke the step waiting for it)
|
|
81
|
+
# pgbus.workflow.wait_timed_out — a waiting step's wait_timeout: passed; its on_timeout
|
|
82
|
+
# applied (the sweep, or TestHelpers#expire_wait)
|
|
83
|
+
# payload: workflow_class, workflow_id, step_name, attempt,
|
|
84
|
+
# action (pause! | cancel! | skip! | run)
|
|
85
|
+
# pgbus.workflow.gauge — emitted by each dispatcher workflow sweep
|
|
86
|
+
# payload: statuses ([{ workflow_class:, status:, count: }]),
|
|
87
|
+
# stuck ([{ workflow_class:, reason:, count: }]), zero-filled
|
|
88
|
+
# across every known class; reasons include waiting_overdue
|
|
57
89
|
#
|
|
58
90
|
module Instrumentation
|
|
59
91
|
module_function
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
],
|
|
5
5
|
"dashboard": {
|
|
6
6
|
"title": "Pgbus",
|
|
7
|
-
"description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\n",
|
|
7
|
+
"description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\nWorkflow step outcomes and run times, and stuck workflows.\n",
|
|
8
8
|
"visuals": [
|
|
9
9
|
{
|
|
10
10
|
"type": "timeseries",
|
|
@@ -393,6 +393,133 @@
|
|
|
393
393
|
"tags": []
|
|
394
394
|
}
|
|
395
395
|
]
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
"type": "timeseries",
|
|
399
|
+
"display": "LINE",
|
|
400
|
+
"title": "Workflow steps per outcome",
|
|
401
|
+
"description": "Workflow step attempts by class, step and outcome (completed, reattempted, skipped, paused, cancelled, finished_early).",
|
|
402
|
+
"line_label": "%workflow_class% - %step% - %outcome%",
|
|
403
|
+
"format": "number",
|
|
404
|
+
"draw_null_as_zero": true,
|
|
405
|
+
"metrics": [
|
|
406
|
+
{
|
|
407
|
+
"name": "pgbus_workflow_steps_total",
|
|
408
|
+
"fields": [
|
|
409
|
+
{
|
|
410
|
+
"field": "COUNTER"
|
|
411
|
+
}
|
|
412
|
+
],
|
|
413
|
+
"tags": [
|
|
414
|
+
{
|
|
415
|
+
"key": "workflow_class",
|
|
416
|
+
"value": "*"
|
|
417
|
+
},
|
|
418
|
+
{
|
|
419
|
+
"key": "step",
|
|
420
|
+
"value": "*"
|
|
421
|
+
},
|
|
422
|
+
{
|
|
423
|
+
"key": "outcome",
|
|
424
|
+
"value": "*"
|
|
425
|
+
}
|
|
426
|
+
]
|
|
427
|
+
}
|
|
428
|
+
]
|
|
429
|
+
},
|
|
430
|
+
{
|
|
431
|
+
"type": "timeseries",
|
|
432
|
+
"display": "LINE",
|
|
433
|
+
"title": "Workflow step run time (seconds)",
|
|
434
|
+
"description": "Mean and p95 run time of each workflow step's body, in seconds.",
|
|
435
|
+
"line_label": "%workflow_class% - %step% - %outcome%",
|
|
436
|
+
"format": "number",
|
|
437
|
+
"draw_null_as_zero": true,
|
|
438
|
+
"metrics": [
|
|
439
|
+
{
|
|
440
|
+
"name": "pgbus_workflow_step_duration_seconds",
|
|
441
|
+
"fields": [
|
|
442
|
+
{
|
|
443
|
+
"field": "MEAN"
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
"field": "P95"
|
|
447
|
+
}
|
|
448
|
+
],
|
|
449
|
+
"tags": [
|
|
450
|
+
{
|
|
451
|
+
"key": "workflow_class",
|
|
452
|
+
"value": "*"
|
|
453
|
+
},
|
|
454
|
+
{
|
|
455
|
+
"key": "step",
|
|
456
|
+
"value": "*"
|
|
457
|
+
},
|
|
458
|
+
{
|
|
459
|
+
"key": "outcome",
|
|
460
|
+
"value": "*"
|
|
461
|
+
}
|
|
462
|
+
]
|
|
463
|
+
}
|
|
464
|
+
]
|
|
465
|
+
},
|
|
466
|
+
{
|
|
467
|
+
"type": "timeseries",
|
|
468
|
+
"display": "LINE",
|
|
469
|
+
"title": "Workflows by status",
|
|
470
|
+
"description": "Workflow runs per class and status, from each dispatcher sweep.",
|
|
471
|
+
"line_label": "%workflow_class% - %status%",
|
|
472
|
+
"format": "number",
|
|
473
|
+
"draw_null_as_zero": true,
|
|
474
|
+
"metrics": [
|
|
475
|
+
{
|
|
476
|
+
"name": "pgbus_workflows",
|
|
477
|
+
"fields": [
|
|
478
|
+
{
|
|
479
|
+
"field": "GAUGE"
|
|
480
|
+
}
|
|
481
|
+
],
|
|
482
|
+
"tags": [
|
|
483
|
+
{
|
|
484
|
+
"key": "workflow_class",
|
|
485
|
+
"value": "*"
|
|
486
|
+
},
|
|
487
|
+
{
|
|
488
|
+
"key": "status",
|
|
489
|
+
"value": "*"
|
|
490
|
+
}
|
|
491
|
+
]
|
|
492
|
+
}
|
|
493
|
+
]
|
|
494
|
+
},
|
|
495
|
+
{
|
|
496
|
+
"type": "timeseries",
|
|
497
|
+
"display": "LINE",
|
|
498
|
+
"title": "Stuck workflows",
|
|
499
|
+
"description": "Ongoing workflow runs that need a person, by class and reason (paused_error, stalled, paused_manual, overdue).",
|
|
500
|
+
"line_label": "%workflow_class% - %reason%",
|
|
501
|
+
"format": "number",
|
|
502
|
+
"draw_null_as_zero": true,
|
|
503
|
+
"metrics": [
|
|
504
|
+
{
|
|
505
|
+
"name": "pgbus_workflows_stuck",
|
|
506
|
+
"fields": [
|
|
507
|
+
{
|
|
508
|
+
"field": "GAUGE"
|
|
509
|
+
}
|
|
510
|
+
],
|
|
511
|
+
"tags": [
|
|
512
|
+
{
|
|
513
|
+
"key": "workflow_class",
|
|
514
|
+
"value": "*"
|
|
515
|
+
},
|
|
516
|
+
{
|
|
517
|
+
"key": "reason",
|
|
518
|
+
"value": "*"
|
|
519
|
+
}
|
|
520
|
+
]
|
|
521
|
+
}
|
|
522
|
+
]
|
|
396
523
|
}
|
|
397
524
|
]
|
|
398
525
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pgbus/integrations/appsignal/span_subscriber"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Integrations
|
|
7
|
+
module Appsignal
|
|
8
|
+
# The `<Handler>#handle` transaction around pgbus.event_processed, and
|
|
9
|
+
# the fallback for an error pgbus.event_failed reports from outside it
|
|
10
|
+
# (an event that could not be built never reaches the span).
|
|
11
|
+
class EventSpan < SpanSubscriber
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
def frame_key = :pgbus_appsignal_event_span
|
|
15
|
+
|
|
16
|
+
def attached_key = :pgbus_appsignal_event_span_error
|
|
17
|
+
|
|
18
|
+
def describe(transaction, payload)
|
|
19
|
+
transaction.set_action_if_nil("#{payload[:handler] || "UnknownHandler"}#handle")
|
|
20
|
+
apply_queue_start(transaction, payload[:published_at])
|
|
21
|
+
transaction.add_tags(handler_tags(payload))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# The processed count is taken at finish whether or not the handler
|
|
25
|
+
# raised, as Metrics::Subscriber does: a raised handler counts as
|
|
26
|
+
# processed and, through pgbus.event_failed, as failed.
|
|
27
|
+
def finished(payload, duration)
|
|
28
|
+
tags = { handler: payload[:handler], routing_key: payload[:routing_key] }
|
|
29
|
+
::Appsignal.add_distribution_value("pgbus_event_duration_ms", duration, tags)
|
|
30
|
+
::Appsignal.increment_counter("pgbus_event_count", 1, tags.merge(status: "processed"))
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def handler_tags(payload)
|
|
34
|
+
{
|
|
35
|
+
"handler" => payload[:handler],
|
|
36
|
+
"routing_key" => payload[:routing_key],
|
|
37
|
+
"attempts" => payload[:read_ct]
|
|
38
|
+
}.compact
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pgbus/integrations/appsignal/span_subscriber"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Integrations
|
|
7
|
+
module Appsignal
|
|
8
|
+
# The `<JobClass>#perform` transaction around pgbus.executor.execute
|
|
9
|
+
# (deserialize + perform + archive), and the fallback for an error
|
|
10
|
+
# pgbus.job_failed reports from outside it.
|
|
11
|
+
class JobSpan < SpanSubscriber
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
def frame_key = :pgbus_appsignal_job_span
|
|
15
|
+
|
|
16
|
+
def attached_key = :pgbus_appsignal_job_span_error
|
|
17
|
+
|
|
18
|
+
def describe(transaction, payload)
|
|
19
|
+
transaction.set_action_if_nil("#{payload[:job_class] || "UnknownJob"}#perform")
|
|
20
|
+
apply_queue_start(transaction, payload[:enqueued_at])
|
|
21
|
+
transaction.add_tags(job_tags(payload))
|
|
22
|
+
transaction.add_params_if_nil { { arguments: payload[:arguments] } } if payload.key?(:arguments)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def finished(payload, duration)
|
|
26
|
+
::Appsignal.add_distribution_value(
|
|
27
|
+
"pgbus_job_duration_ms",
|
|
28
|
+
duration,
|
|
29
|
+
{ queue: payload[:queue], job_class: payload[:job_class] }
|
|
30
|
+
)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def job_tags(payload)
|
|
34
|
+
tags = {
|
|
35
|
+
"queue" => payload[:queue],
|
|
36
|
+
"job_class" => payload[:job_class],
|
|
37
|
+
"attempts" => payload[:read_ct]
|
|
38
|
+
}
|
|
39
|
+
tags["active_job_id"] = payload[:job_id] if payload[:job_id]
|
|
40
|
+
tags["provider_job_id"] = payload[:provider_job_id] if payload[:provider_job_id]
|
|
41
|
+
tags["request_id"] = payload[:provider_job_id] || payload[:job_id]
|
|
42
|
+
tags.compact
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Integrations
|
|
7
|
+
module Appsignal
|
|
8
|
+
# The AppSignal transaction of one unit of work: a job (JobSpan, on
|
|
9
|
+
# pgbus.executor.execute) or an event handler (EventSpan, on
|
|
10
|
+
# pgbus.event_processed).
|
|
11
|
+
#
|
|
12
|
+
# An ActiveSupport::Notifications listener that responds to #start and
|
|
13
|
+
# #finish is called on both sides of the instrumented block (the Evented
|
|
14
|
+
# subscriber). A finish-only block subscriber runs after the block has
|
|
15
|
+
# returned, so the transaction it opened was never current while the job
|
|
16
|
+
# ran: job code had nothing to attach to, and pgbus.job_failed, fired
|
|
17
|
+
# later, found no transaction to put its error on (issue #552). Here
|
|
18
|
+
# #start opens the transaction before the block runs, so job and handler
|
|
19
|
+
# code, and the Rails error reporter, find it current; #finish records
|
|
20
|
+
# the exception the block raised (the instrumenter puts it in
|
|
21
|
+
# payload[:exception_object]) and completes the transaction.
|
|
22
|
+
#
|
|
23
|
+
# Like AppSignal's own ActiveJob hook, a transaction that is already
|
|
24
|
+
# current is reused and left to its owner to complete.
|
|
25
|
+
#
|
|
26
|
+
# The frame lives in Thread.current[], which is fiber-local, so start and
|
|
27
|
+
# finish pair up per job under execution_mode: :async, the same way
|
|
28
|
+
# AppSignal keeps its current transaction. A span opened inside another
|
|
29
|
+
# of its kind (a handler calling another handler's #process) keeps the
|
|
30
|
+
# enclosing frame and restores it at finish, so the outer span still
|
|
31
|
+
# completes its transaction. An instance holds no state of its own.
|
|
32
|
+
# Nothing an AppSignal call raises reaches the job: it is logged at warn
|
|
33
|
+
# level.
|
|
34
|
+
#
|
|
35
|
+
# Subclasses define #frame_key and #attached_key (the fiber-local slots),
|
|
36
|
+
# #describe(transaction, payload) (action, queue start, tags) and
|
|
37
|
+
# #finished(payload, duration_ms) (the span's metrics).
|
|
38
|
+
class SpanSubscriber
|
|
39
|
+
BACKGROUND_JOB = "background_job"
|
|
40
|
+
|
|
41
|
+
# owned: this span created the transaction, so it completes it.
|
|
42
|
+
# previous: the enclosing span's frame, restored at finish.
|
|
43
|
+
Frame = Struct.new(:started_at, :owned, :previous)
|
|
44
|
+
|
|
45
|
+
def start(_name, _id, payload)
|
|
46
|
+
frame = Thread.current[frame_key] = Frame.new(monotonic_now, false, Thread.current[frame_key])
|
|
47
|
+
guard do
|
|
48
|
+
frame.owned = !::Appsignal::Transaction.current?
|
|
49
|
+
transaction = frame.owned ? ::Appsignal::Transaction.create(BACKGROUND_JOB) : ::Appsignal::Transaction.current
|
|
50
|
+
describe(transaction, payload)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def finish(_name, _id, payload)
|
|
55
|
+
frame = Thread.current[frame_key]
|
|
56
|
+
guard do
|
|
57
|
+
Thread.current[attached_key] = attach(payload[:exception_object])
|
|
58
|
+
finished(payload, monotonic_now - frame.started_at)
|
|
59
|
+
end
|
|
60
|
+
ensure
|
|
61
|
+
guard do
|
|
62
|
+
Thread.current[frame_key] = frame.previous
|
|
63
|
+
::Appsignal::Transaction.complete_current! if frame.owned
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# pgbus.job_failed and pgbus.event_failed fire after #finish. An error
|
|
68
|
+
# the span recorded is already on its transaction. One raised before
|
|
69
|
+
# the span opened (a message body that is not JSON, an event that
|
|
70
|
+
# cannot be built, a dead-letter move that failed) had no transaction,
|
|
71
|
+
# so it is reported on one of its own, named and tagged like the span's
|
|
72
|
+
# (or added to a transaction that is current, which is left as it is).
|
|
73
|
+
def report_unattached(error, payload)
|
|
74
|
+
attached = Thread.current[attached_key]
|
|
75
|
+
Thread.current[attached_key] = nil
|
|
76
|
+
return if error.nil? || error.equal?(attached)
|
|
77
|
+
return ::Appsignal::Transaction.current.set_error(error) if ::Appsignal::Transaction.current?
|
|
78
|
+
|
|
79
|
+
::Appsignal.report_error(error) do |transaction|
|
|
80
|
+
transaction.set_namespace(BACKGROUND_JOB)
|
|
81
|
+
describe(transaction, payload)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
# Returns the exception it recorded, nil when there was none.
|
|
88
|
+
def attach(error)
|
|
89
|
+
return unless error && ::Appsignal::Transaction.current?
|
|
90
|
+
|
|
91
|
+
::Appsignal::Transaction.current.set_error(error)
|
|
92
|
+
error
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# AppSignal expects queue-start as Unix epoch milliseconds. Pgbus
|
|
96
|
+
# carries it as either an ISO-8601 String or a Time — both happen
|
|
97
|
+
# in practice (executor passes the JSON string, handler passes a
|
|
98
|
+
# parsed Time).
|
|
99
|
+
def apply_queue_start(transaction, value)
|
|
100
|
+
return unless value
|
|
101
|
+
|
|
102
|
+
millis =
|
|
103
|
+
case value
|
|
104
|
+
when Time
|
|
105
|
+
(value.to_f * 1_000).to_i
|
|
106
|
+
when String
|
|
107
|
+
(Time.parse(value).to_f * 1_000).to_i
|
|
108
|
+
when Numeric
|
|
109
|
+
value.to_i
|
|
110
|
+
end
|
|
111
|
+
transaction.set_queue_start(millis) if millis
|
|
112
|
+
rescue ArgumentError
|
|
113
|
+
# Unparseable timestamp — skip rather than blow up.
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def guard
|
|
117
|
+
yield
|
|
118
|
+
rescue StandardError => e
|
|
119
|
+
Pgbus.logger.warn { "[Pgbus::AppSignal] subscriber error: #{e.class}: #{e.message}" }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def monotonic_now
|
|
123
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :float_millisecond)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|