pgbus 0.19.4 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +9 -0
  3. data/README.md +97 -7
  4. data/Rakefile +6 -1
  5. data/app/controllers/pgbus/dashboard_controller.rb +2 -0
  6. data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
  7. data/app/controllers/pgbus/workflows_controller.rb +197 -0
  8. data/app/frontend/pgbus/modules/charts.js +148 -4
  9. data/app/frontend/pgbus/modules/charts.min.js +1 -1
  10. data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
  11. data/app/frontend/pgbus/style.css +1 -1
  12. data/app/helpers/pgbus/button_helper.rb +4 -2
  13. data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
  14. data/app/helpers/pgbus/workflows_helper.rb +165 -0
  15. data/app/models/pgbus/blocked_execution.rb +12 -0
  16. data/app/models/pgbus/uniqueness_key.rb +88 -3
  17. data/app/models/pgbus/workflow_entry.rb +172 -0
  18. data/app/models/pgbus/workflow_event.rb +43 -0
  19. data/app/models/pgbus/workflow_signal.rb +77 -0
  20. data/app/models/pgbus/workflow_step.rb +149 -0
  21. data/app/views/layouts/pgbus/application.html.erb +2 -0
  22. data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
  23. data/app/views/pgbus/workflows/_actions.html.erb +59 -0
  24. data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
  25. data/app/views/pgbus/workflows/_filters.html.erb +29 -0
  26. data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
  27. data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
  28. data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
  29. data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
  30. data/app/views/pgbus/workflows/_timeline.html.erb +49 -0
  31. data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
  32. data/app/views/pgbus/workflows/attention.html.erb +88 -0
  33. data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
  34. data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
  35. data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
  36. data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
  37. data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
  38. data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
  39. data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
  40. data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
  41. data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
  42. data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
  43. data/app/views/pgbus/workflows/index.html.erb +15 -0
  44. data/app/views/pgbus/workflows/not_found.html.erb +5 -0
  45. data/app/views/pgbus/workflows/show.html.erb +93 -0
  46. data/config/i18n-tasks.yml +6 -0
  47. data/config/locales/da.yml +445 -0
  48. data/config/locales/de.yml +445 -0
  49. data/config/locales/en.yml +445 -0
  50. data/config/locales/es.yml +445 -0
  51. data/config/locales/fi.yml +445 -0
  52. data/config/locales/fr.yml +445 -0
  53. data/config/locales/it.yml +445 -0
  54. data/config/locales/ja.yml +445 -0
  55. data/config/locales/nb.yml +445 -0
  56. data/config/locales/nl.yml +445 -0
  57. data/config/locales/pt.yml +445 -0
  58. data/config/locales/sv.yml +445 -0
  59. data/config/routes.rb +26 -0
  60. data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
  61. data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
  62. data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
  63. data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
  64. data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
  65. data/lib/pgbus/active_job/adapter.rb +81 -3
  66. data/lib/pgbus/active_job/delivery.rb +63 -0
  67. data/lib/pgbus/active_job/executor.rb +20 -8
  68. data/lib/pgbus/concurrency/blocked_execution.rb +14 -6
  69. data/lib/pgbus/configuration/workflow_settings.rb +99 -0
  70. data/lib/pgbus/configuration.rb +64 -4
  71. data/lib/pgbus/dead_letter_header.rb +9 -1
  72. data/lib/pgbus/generators/migration_detector.rb +40 -0
  73. data/lib/pgbus/instrumentation.rb +33 -0
  74. data/lib/pgbus/integrations/appsignal/dashboard.json +157 -1
  75. data/lib/pgbus/integrations/appsignal/subscriber.rb +16 -1
  76. data/lib/pgbus/mcp/rack_app.rb +7 -2
  77. data/lib/pgbus/mcp/runner.rb +2 -1
  78. data/lib/pgbus/mcp/serialization.rb +25 -0
  79. data/lib/pgbus/mcp/server.rb +53 -10
  80. data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
  81. data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
  82. data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
  83. data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
  84. data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
  85. data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
  86. data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
  87. data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
  88. data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
  89. data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
  90. data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
  91. data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
  92. data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
  93. data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
  94. data/lib/pgbus/mcp/write_tool.rb +134 -0
  95. data/lib/pgbus/mcp.rb +15 -0
  96. data/lib/pgbus/metrics/backends/statsd.rb +6 -0
  97. data/lib/pgbus/metrics/subscriber.rb +14 -1
  98. data/lib/pgbus/metrics/workflow_metrics.rb +75 -0
  99. data/lib/pgbus/process/consumer.rb +2 -1
  100. data/lib/pgbus/process/dispatcher.rb +39 -3
  101. data/lib/pgbus/process/worker.rb +10 -1
  102. data/lib/pgbus/recurring/schedule.rb +1 -1
  103. data/lib/pgbus/streams/presence.rb +65 -36
  104. data/lib/pgbus/uniqueness.rb +23 -6
  105. data/lib/pgbus/version.rb +1 -1
  106. data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
  107. data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
  108. data/lib/pgbus/web/data_source/workflow_list.rb +291 -0
  109. data/lib/pgbus/web/data_source.rb +126 -15
  110. data/lib/pgbus/web/workflow_action_plan.rb +62 -0
  111. data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
  112. data/lib/pgbus/web/workflow_insights_report.rb +171 -0
  113. data/lib/pgbus/web/workflow_payload.rb +36 -0
  114. data/lib/pgbus/web/workflow_query.rb +82 -0
  115. data/lib/pgbus/web/workflow_state.rb +89 -0
  116. data/lib/pgbus/web/workflow_timeline.rb +130 -0
  117. data/lib/pgbus/web/workflow_window.rb +51 -0
  118. data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
  119. data/lib/pgbus/workflow/dsl.rb +105 -0
  120. data/lib/pgbus/workflow/error_capture.rb +63 -0
  121. data/lib/pgbus/workflow/exception_policy.rb +170 -0
  122. data/lib/pgbus/workflow/fan_out.rb +157 -0
  123. data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
  124. data/lib/pgbus/workflow/flow_control.rb +94 -0
  125. data/lib/pgbus/workflow/instance.rb +183 -0
  126. data/lib/pgbus/workflow/outside_verbs.rb +138 -0
  127. data/lib/pgbus/workflow/payload.rb +53 -0
  128. data/lib/pgbus/workflow/registry.rb +53 -0
  129. data/lib/pgbus/workflow/removed_step.rb +14 -0
  130. data/lib/pgbus/workflow/runner.rb +218 -0
  131. data/lib/pgbus/workflow/signal_matcher.rb +53 -0
  132. data/lib/pgbus/workflow/signals.rb +132 -0
  133. data/lib/pgbus/workflow/states.rb +119 -0
  134. data/lib/pgbus/workflow/step_context.rb +58 -0
  135. data/lib/pgbus/workflow/step_definition.rb +133 -0
  136. data/lib/pgbus/workflow/step_execution.rb +242 -0
  137. data/lib/pgbus/workflow/step_job.rb +28 -0
  138. data/lib/pgbus/workflow/step_outcome.rb +151 -0
  139. data/lib/pgbus/workflow/steps.rb +103 -0
  140. data/lib/pgbus/workflow/sweep.rb +156 -0
  141. data/lib/pgbus/workflow/test_helpers.rb +152 -0
  142. data/lib/pgbus/workflow/wait_timeout.rb +94 -0
  143. data/lib/pgbus/workflow/waiting.rb +82 -0
  144. data/lib/pgbus/workflow.rb +183 -0
  145. data/sig/pgbus/configuration.rbs +23 -0
  146. metadata +92 -1
@@ -0,0 +1,134 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module MCP
5
+ # Base class for the opt-in write tools (issue #550). Deliberately not a
6
+ # BaseTool subclass: no write tool can inherit the read-only annotations,
7
+ # and no read tool can become a write tool by changing its superclass.
8
+ # Server registers these only when config.mcp_write_tools is true.
9
+ #
10
+ # Every call (#perform): the run loaded (unknown: ok: false), the tool's
11
+ # inputs and a reason (1-500 characters after trimming) checked,
12
+ # expected_status checked when given, the verb checked against the run's
13
+ # available_actions, then the same Web::DataSource action the dashboard
14
+ # uses, recorded as actor "mcp" (or "mcp:<client name>"). The answer is
15
+ # { ok:, reason:, workflow: } with the run as it is after the call,
16
+ # payloads always redacted; a refusal changes nothing and records nothing.
17
+ #
18
+ # The two checks read the run before acting, so they are advisory: the
19
+ # action's guarded transition under the run's row lock decides, and its
20
+ # code (:not_allowed, with the status the run is in now) is passed on.
21
+ class WriteTool < ::MCP::Tool
22
+ ANNOTATIONS = { read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: false }.freeze
23
+ REASON_LIMIT = 500
24
+ CLIENT_NAME_LIMIT = 100
25
+ ACTOR = "mcp"
26
+ # What an unexpected failure answers: its text can carry internals
27
+ # (hosts, SQL, values), and Web::DataSource has already logged it.
28
+ FAILED = "the action failed unexpectedly; the error is in the pgbus log"
29
+
30
+ annotations(**ANNOTATIONS)
31
+
32
+ class << self
33
+ # MCP::Tool.inherited resets @annotations_value on every subclass, so
34
+ # fall back to the nearest ancestor's (as BaseTool does).
35
+ def annotations_value
36
+ super || (superclass.respond_to?(:annotations_value) ? superclass.annotations_value : nil)
37
+ end
38
+
39
+ # For a verb whose repeat can never act again (cancel: a cancelled run
40
+ # stays cancelled). Not skip or resume at: a retry can act again once
41
+ # the run has parked or paused again.
42
+ def idempotent!
43
+ annotations(**ANNOTATIONS, idempotent_hint: true)
44
+ end
45
+
46
+ # The input_schema every write tool takes, plus the tool's own fields.
47
+ def workflow_input_schema(properties: {}, required: [])
48
+ {
49
+ properties: {
50
+ workflow_id: { type: "string", description: "The run's workflow_id (a UUID)." },
51
+ reason: { type: "string", minLength: 1, maxLength: REASON_LIMIT,
52
+ description: "Why, in 1-#{REASON_LIMIT} characters. Recorded on the run's audit trail." },
53
+ expected_status: {
54
+ type: "string", enum: Workflow::States::WORKFLOW,
55
+ description: "The status you read. If the run is in another one when the call starts: ok: false, nothing done."
56
+ },
57
+ **properties
58
+ },
59
+ required: %w[workflow_id reason] + required
60
+ }
61
+ end
62
+
63
+ # The tool description: what it does, the transition it makes, and the
64
+ # opt-in. Frozen: it is class state (the conformance audit checks).
65
+ def verb_description(verb, summary)
66
+ transition = Workflow::States.fetch(verb)
67
+ <<~DESC.freeze
68
+ #{summary} Allowed from: #{transition.from.join(", ")}. What happens: #{transition.effect}. A write tool: it exists only when the server was built with config.mcp_write_tools = true. reason is required and recorded on the run's audit trail with actor "mcp" (or "mcp:<client name>" when the transport knows the client); pass expected_status (the status you read with pgbus_workflow): a run that has moved on by the time the call starts answers ok: false and nothing is done (the check runs just before the action, whose transition is guarded again under the run's row lock). Answers { ok, reason, workflow } with the run as it is after the call; a refused call changes nothing and records nothing (a transition refused under the lock adds code and status, a disallowed verb available_actions); arguments, outputs, signal payloads and error text are redacted.
69
+ DESC
70
+ end
71
+
72
+ # Runs the checks, then yields (data_source, actor:, reason:) for the
73
+ # DataSource action, whose { ok:, ... } becomes the answer. +invalid+
74
+ # is the tool's own input refusal (a blank step, an oversized
75
+ # payload), answered like every other refusal: with the run as it is.
76
+ def perform(verb, workflow_id:, reason:, expected_status:, server_context:, invalid: nil)
77
+ data_source = BaseTool.data_source_from(server_context)
78
+ detail = data_source.workflow_detail(workflow_id)
79
+ return answer(ok: false, reason: "workflow #{workflow_id} not found", workflow: nil) unless detail
80
+
81
+ note = reason.to_s.strip
82
+ refusal = input_refusal(note, invalid) || guard_refusal(verb, detail, expected_status)
83
+ return answer(**refusal, workflow: present(detail)) if refusal
84
+
85
+ result = outcome(yield(data_source, actor: actor(server_context), reason: note))
86
+ answer(**result, workflow: present(data_source.workflow_detail(workflow_id)))
87
+ end
88
+
89
+ def answer(**value) = BaseTool.json_response(value)
90
+
91
+ private
92
+
93
+ def input_refusal(note, invalid)
94
+ unless note.length.between?(1, REASON_LIMIT)
95
+ return { ok: false, reason: "reason must be 1-#{REASON_LIMIT} characters after trimming whitespace" }
96
+ end
97
+
98
+ { ok: false, reason: invalid } if invalid
99
+ end
100
+
101
+ # The DataSource action's answer, ok and reason first; an unexpected
102
+ # failure's text replaced (FAILED).
103
+ def outcome(result)
104
+ reason = result[:code] == :failed ? FAILED : result[:reason]
105
+ { ok: result[:ok], reason: reason, **result.except(:ok, :reason) }
106
+ end
107
+
108
+ def guard_refusal(verb, detail, expected_status)
109
+ status = detail[:status].to_s
110
+ if expected_status && expected_status.to_s != status
111
+ return { ok: false, reason: "workflow #{detail[:workflow_id]} is #{status}, not #{expected_status}" }
112
+ end
113
+
114
+ allowed = Array(detail[:available_actions]).map(&:to_s)
115
+ return if allowed.include?(verb.to_s)
116
+
117
+ { ok: false, reason: "cannot #{verb.to_s.tr("_", " ")} a #{status} workflow", available_actions: allowed }
118
+ end
119
+
120
+ def present(detail) = detail && WorkflowPayloads.detail(detail, allowed: false)
121
+
122
+ # "mcp", or "mcp:<name>" when the transport knows the client (an
123
+ # MCP::ServerContext on mcp >= 1.x; a plain Hash context has none).
124
+ # The name is client-supplied: control characters dropped, length capped.
125
+ def actor(server_context)
126
+ info = server_context.client_info if server_context.respond_to?(:client_info)
127
+ name = info.is_a?(Hash) ? (info[:name] || info["name"]) : nil
128
+ name = name.to_s.gsub(/[^[:print:]]/, "").strip[0, CLIENT_NAME_LIMIT]
129
+ name.empty? ? ACTOR : "#{ACTOR}:#{name}"
130
+ end
131
+ end
132
+ end
133
+ end
134
+ end
data/lib/pgbus/mcp.rb CHANGED
@@ -14,6 +14,7 @@ module Pgbus
14
14
  REQUIRES = %w[
15
15
  pgbus/mcp/redactor
16
16
  pgbus/mcp/base_tool
17
+ pgbus/mcp/serialization
17
18
  pgbus/mcp/health_analyzer
18
19
  pgbus/mcp/tools/health_tool
19
20
  pgbus/mcp/tools/queues_tool
@@ -29,6 +30,20 @@ module Pgbus
29
30
  pgbus/mcp/tools/stats_tool
30
31
  pgbus/mcp/tools/recurring_tool
31
32
  pgbus/mcp/tools/deploy_tasks_tool
33
+ pgbus/mcp/workflow_payloads
34
+ pgbus/mcp/tools/workflows_tool
35
+ pgbus/mcp/tools/workflow_detail_tool
36
+ pgbus/mcp/tools/workflows_attention_tool
37
+ pgbus/mcp/workflow_insights_payload
38
+ pgbus/mcp/tools/workflow_definitions_tool
39
+ pgbus/mcp/tools/workflow_stats_tool
40
+ pgbus/mcp/write_tool
41
+ pgbus/mcp/tools/workflow_resume_tool
42
+ pgbus/mcp/tools/workflow_cancel_tool
43
+ pgbus/mcp/tools/workflow_skip_step_tool
44
+ pgbus/mcp/tools/workflow_resume_at_tool
45
+ pgbus/mcp/tools/workflow_run_now_tool
46
+ pgbus/mcp/tools/workflow_signal_tool
32
47
  pgbus/mcp/server
33
48
  pgbus/mcp/runner
34
49
  pgbus/mcp/rack_app
@@ -11,6 +11,10 @@ module Pgbus
11
11
  # gauge -> "name:value|g"
12
12
  # histogram -> "name:value|ms"
13
13
  #
14
+ # StatsD reads |ms values as milliseconds, so a histogram named
15
+ # *_seconds (the Prometheus convention, e.g.
16
+ # pgbus_workflow_step_duration_seconds) is sent as *_ms, times 1000.
17
+ #
14
18
  # Tags are appended as DogStatsD `|#k:v,k:v`. No gem dependency — a plain
15
19
  # UDPSocket. UDP send is fire-and-forget; any socket error is logged and
16
20
  # swallowed so a metrics hiccup never propagates into the producer thread.
@@ -32,6 +36,8 @@ module Pgbus
32
36
  end
33
37
 
34
38
  def histogram(name, value, tags = {})
39
+ return emit("#{name.delete_suffix("_seconds")}_ms", value * 1000.0, "ms", tags) if name.end_with?("_seconds")
40
+
35
41
  emit(name, value, "ms", tags)
36
42
  end
37
43
 
@@ -73,7 +73,8 @@ module Pgbus
73
73
  subscribe("pgbus.recurring.enqueue") { |event| on_recurring_enqueue(event) },
74
74
  subscribe("pgbus.worker.recycle") { |event| on_worker_recycle(event) },
75
75
  subscribe("pgbus.consumer.recycle") { |event| on_consumer_recycle(event) },
76
- subscribe("pgbus.client.pool") { |event| on_client_pool(event) }
76
+ subscribe("pgbus.client.pool") { |event| on_client_pool(event) },
77
+ *WorkflowMetrics::EVENTS.map { |name| subscribe(name) { |event| on_workflow(event) } }
77
78
  ].freeze
78
79
  end
79
80
 
@@ -251,6 +252,18 @@ module Pgbus
251
252
  backend.gauge("#{METRIC_PREFIX}pool_available", payload[:available], tags)
252
253
  end
253
254
 
255
+ # ── Workflows (issue #540) ────────────────────────────────────────
256
+
257
+ def on_workflow(event)
258
+ WorkflowMetrics.samples(event.name, event.payload).each do |kind, name, value, tags|
259
+ case kind
260
+ when :histogram then backend.histogram(name, value, tags)
261
+ when :counter then backend.increment(name, value, tags)
262
+ when :gauge then backend.gauge(name, value, tags)
263
+ end
264
+ end
265
+ end
266
+
254
267
  def hostname
255
268
  Socket.gethostname
256
269
  end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Metrics
5
+ # The pgbus.workflow.* event → metric mapping (issue #540), shared by
6
+ # Metrics::Subscriber (Prometheus, StatsD) and the AppSignal subscriber so
7
+ # both backends report the same names and labels:
8
+ #
9
+ # pgbus_workflow_step_duration_seconds{workflow_class, step, outcome} histogram
10
+ # pgbus_workflow_steps_total{workflow_class, step, outcome} counter
11
+ # pgbus_workflows_finished_total{workflow_class} counter
12
+ # pgbus_workflows_paused_total{workflow_class, error_class} counter (error_class absent for a manual pause)
13
+ # pgbus_workflows_cancelled_total{workflow_class} counter
14
+ # pgbus_workflow_stalls_total{workflow_class, step, detection} counter (detection: redelivery | sweep)
15
+ # pgbus_workflow_wait_overdue_total{workflow_class, step} counter (a late notice of an on_timeout: :notify wait)
16
+ # pgbus_workflows{workflow_class, status} gauge, from each sweep
17
+ # pgbus_workflows_stuck{workflow_class, reason} gauge, from each sweep
18
+ #
19
+ # Label values come from code (class, step and exception names), so
20
+ # their cardinality is bounded; workflow ids never become labels.
21
+ module WorkflowMetrics
22
+ EVENTS = %w[
23
+ pgbus.workflow.step pgbus.workflow_finished pgbus.workflow_paused pgbus.workflow_cancelled
24
+ pgbus.workflow.stall_detected pgbus.workflow.wait_timed_out pgbus.workflow.gauge
25
+ ].freeze
26
+
27
+ module_function
28
+
29
+ # [[kind, name, value, tags], ...] for one event; kind is :histogram,
30
+ # :counter or :gauge.
31
+ def samples(event_name, payload)
32
+ klass = payload[:workflow_class]
33
+ case event_name
34
+ when "pgbus.workflow.step" then step_samples(klass, payload)
35
+ when "pgbus.workflow_finished" then [counter("pgbus_workflows_finished_total", workflow_class: klass)]
36
+ when "pgbus.workflow_paused"
37
+ [counter("pgbus_workflows_paused_total", workflow_class: klass, error_class: payload[:error_class])]
38
+ when "pgbus.workflow_cancelled" then [counter("pgbus_workflows_cancelled_total", workflow_class: klass)]
39
+ when "pgbus.workflow.stall_detected"
40
+ [counter("pgbus_workflow_stalls_total", workflow_class: klass, step: payload[:step_name], detection: payload[:detection])]
41
+ when "pgbus.workflow.wait_timed_out" then wait_timed_out_samples(klass, payload)
42
+ when "pgbus.workflow.gauge" then gauge_samples(payload)
43
+ else []
44
+ end
45
+ end
46
+
47
+ def step_samples(klass, payload)
48
+ tags = { workflow_class: klass, step: payload[:step_name], outcome: payload[:outcome] }.compact
49
+ samples = [[:counter, "pgbus_workflow_steps_total", 1, tags]]
50
+ samples.unshift([:histogram, "pgbus_workflow_step_duration_seconds", payload[:duration], tags]) if payload[:duration]
51
+ samples
52
+ end
53
+
54
+ # Only :notify keeps waiting past the deadline; every other on_timeout
55
+ # ends the wait and is counted by the step or run it ends.
56
+ def wait_timed_out_samples(klass, payload)
57
+ return [] unless payload[:action] == "notify"
58
+
59
+ [counter("pgbus_workflow_wait_overdue_total", workflow_class: klass, step: payload[:step_name])]
60
+ end
61
+
62
+ def gauge_samples(payload)
63
+ statuses = Array(payload[:statuses]).map do |row|
64
+ [:gauge, "pgbus_workflows", row[:count], { workflow_class: row[:workflow_class], status: row[:status] }]
65
+ end
66
+ stuck = Array(payload[:stuck]).map do |row|
67
+ [:gauge, "pgbus_workflows_stuck", row[:count], { workflow_class: row[:workflow_class], reason: row[:reason] }]
68
+ end
69
+ statuses + stuck
70
+ end
71
+
72
+ def counter(name, **tags) = [:counter, name, 1, tags.compact]
73
+ end
74
+ end
75
+ end
@@ -296,7 +296,8 @@ module Pgbus
296
296
  last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
297
297
  headers = DeadLetterHeader.build(
298
298
  existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "consumer",
299
- source_queue: config.queue_name(queue_name), attempts: message.read_ct.to_i,
299
+ source_queue: config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
300
+ source_enqueued_at: message.enqueued_at, attempts: message.read_ct.to_i,
300
301
  max_retries: config.max_retries, error: last_error
301
302
  )
302
303
  Pgbus.client.move_to_dead_letter(queue_name, message, headers: headers)
@@ -19,6 +19,7 @@ module Pgbus
19
19
  JOB_LOCK_CLEANUP_INTERVAL = 300 # Run job lock cleanup every 5 minutes
20
20
  STATS_CLEANUP_INTERVAL = 3600 # Run stats cleanup every hour
21
21
  ORPHAN_STREAM_SWEEP_INTERVAL = 3600 # Run orphan stream sweep every hour
22
+ WORKFLOW_CLEANUP_INTERVAL = 3600 # Delete finished/cancelled workflows past workflow_retention every hour
22
23
  TABLE_MAINTENANCE_INTERVAL = Pgbus::TableMaintenance::MAINTENANCE_INTERVAL
23
24
 
24
25
  # Page size for archive compaction. Each cycle deletes up to this
@@ -57,6 +58,7 @@ module Pgbus
57
58
  @last_stream_archive_compaction_at
58
59
  @last_outbox_cleanup_at @last_job_lock_cleanup_at @last_stats_cleanup_at
59
60
  @last_orphan_stream_sweep_at @last_table_maintenance_at
61
+ @last_workflow_sweep_at @last_workflow_cleanup_at
60
62
  ].freeze
61
63
 
62
64
  # Outcome of a single maintenance task in a cycle. Lets run_maintenance
@@ -247,6 +249,8 @@ module Pgbus
247
249
  sweep_interval = config.streams_orphan_sweep_interval
248
250
  results << run_if_due(now, :@last_orphan_stream_sweep_at, sweep_interval) { sweep_orphan_streams } if sweep_interval
249
251
  results << run_if_due(now, :@last_table_maintenance_at, TABLE_MAINTENANCE_INTERVAL) { run_table_maintenance }
252
+ results << run_if_due(now, :@last_workflow_sweep_at, config.workflow_sweep_interval) { sweep_workflows }
253
+ results << run_if_due(now, :@last_workflow_cleanup_at, WORKFLOW_CLEANUP_INTERVAL) { cleanup_workflows }
250
254
  results
251
255
  end
252
256
 
@@ -388,6 +392,28 @@ module Pgbus
388
392
  e
389
393
  end
390
394
 
395
+ # Stalled and overdue workflow steps, timed-out waits, and the workflow gauges.
396
+ def sweep_workflows
397
+ summary = Workflow::Sweep.repair(config: config)
398
+ return unless summary && summary.values_at(:stalled, :resent, :timed_out, :fan_outs).compact.any?(&:positive?)
399
+
400
+ Pgbus.logger.info do
401
+ "[Pgbus] Workflow sweep: #{summary[:stalled]} stalled step(s) repaired, #{summary[:resent]} overdue step(s) re-sent, " \
402
+ "#{summary[:timed_out].to_i} wait(s) timed out, #{summary[:fan_outs].to_i} fan-out(s) resolved"
403
+ end
404
+ rescue StandardError => e
405
+ log_maintenance_failure("Workflow sweep", e)
406
+ e
407
+ end
408
+
409
+ def cleanup_workflows
410
+ deleted = Workflow::Sweep.cleanup(retention: config.workflow_retention)
411
+ Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} finished workflows" } if deleted.positive?
412
+ rescue StandardError => e
413
+ log_maintenance_failure("Workflow cleanup", e)
414
+ e
415
+ end
416
+
391
417
  def cleanup_stats
392
418
  retention = config.stats_retention
393
419
  return unless retention&.positive?
@@ -479,12 +505,22 @@ module Pgbus
479
505
  # Unbound rows (synthetic "pending" queue and/or msg_id=0) must not probe
480
506
  # pgmq.q_<prefix>_pending — that table does not exist, message_exists?
481
507
  # returns nil, and the reaper would keep the lock forever (issue #418).
482
- # Scan every live queue for the payload key instead.
508
+ # Scan every live queue for the payload key instead. A job parked behind
509
+ # its concurrency slot is in no queue: its key counts as live while it
510
+ # is parked (issue #564). The parked jobs are asked first: a promotion
511
+ # sends the job's message before its parked row's delete commits (or
512
+ # both in one transaction), so a job promoted between the two reads is
513
+ # still parked for the first or already queued for the second. Read the
514
+ # other way round it could be in neither.
483
515
  def gone_unbound_locks(unbound)
484
516
  return [] if unbound.empty?
485
517
 
486
- present = Pgbus.client.uniqueness_keys_present(unbound.map(&:lock_key))
487
- unbound.reject { |key| present.include?(key.lock_key) }
518
+ parked = BlockedExecution.parked_uniqueness_keys(unbound.map(&:lock_key))
519
+ unparked = unbound.reject { |key| parked.include?(key.lock_key) }
520
+ return [] if unparked.empty?
521
+
522
+ present = Pgbus.client.uniqueness_keys_present(unparked.map(&:lock_key))
523
+ unparked.reject { |key| present.include?(key.lock_key) }
488
524
  end
489
525
 
490
526
  # Returns true if the message referenced by this bound lock is definitely
@@ -407,12 +407,21 @@ module Pgbus
407
407
  # so we also pass `limit: qty` to cap the total across all queues —
408
408
  # otherwise we get `queue_count * qty` messages and overflow the
409
409
  # execution pool, crashing the worker fork (issue #123).
410
+ #
411
+ # The logical name is the configured one the table was made from, not
412
+ # the table name with its prefix cut: a name normalized for its table
413
+ # ("orders-handler" → "orders_handler") would otherwise reach the
414
+ # executor spelled unlike the name its jobs' uniqueness locks are bound
415
+ # under, and unlike every other fetch path (issue #564).
410
416
  def fetch_multi(active_queues, qty)
411
417
  messages = Pgbus.client.read_multi(active_queues, qty: qty, limit: qty) || []
418
+ return messages if messages.empty?
419
+
412
420
  prefix = "#{config.queue_prefix}_"
421
+ configured = active_queues.to_h { |q| [config.queue_name(q), q] }
413
422
 
414
423
  messages.map do |m|
415
- logical = m.queue_name&.delete_prefix(prefix) || active_queues.first
424
+ logical = configured[m.queue_name] || m.queue_name&.delete_prefix(prefix) || active_queues.first
416
425
  [logical, m]
417
426
  end
418
427
  end
@@ -143,7 +143,7 @@ module Pgbus
143
143
  key = resolve_uniqueness_key(job_class, config, task)
144
144
  return nil unless key
145
145
 
146
- acquired = UniquenessKey.acquire!(key, queue_name: resolve_queue(task), msg_id: 0)
146
+ acquired = UniquenessKey.acquire!(key, queue_name: Uniqueness.lock_queue_name(resolve_queue(task)), msg_id: 0)
147
147
 
148
148
  if acquired
149
149
  key
@@ -72,6 +72,7 @@ module Pgbus
72
72
  @schema_mutex = Mutex.new
73
73
  @seats_supported = nil
74
74
  @probed_at = nil
75
+ @schema_generation = 0
75
76
 
76
77
  class << self
77
78
  # True when pgbus_presence_members has the seat columns (the
@@ -81,17 +82,29 @@ module Pgbus
81
82
  # The answer is read and written under @schema_mutex only (issue
82
83
  # #527). A successful probe is kept as above; a probe that raises is
83
84
  # not, so the next call probes again.
85
+ #
86
+ # The probe itself runs outside @schema_mutex (issue #566): it waits
87
+ # for the adapter's lock, and a request inside a transaction holds
88
+ # that lock while it asks seats_supported?. Two threads may probe at
89
+ # once; a true answer one of them stored is never taken back, and a
90
+ # probe that a reset_schema_cache! overtook stores nothing.
84
91
  def seats_supported?
92
+ cached, generation = @schema_mutex.synchronize do
93
+ [(@seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)), @schema_generation]
94
+ end
95
+ return cached unless cached.nil?
96
+
97
+ rows = exec_params(<<~SQL, [])
98
+ SELECT 1 AS present FROM information_schema.columns
99
+ WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
100
+ AND table_schema = ANY (current_schemas(false))
101
+ SQL
102
+ supported = !rows.to_a.empty?
85
103
  @schema_mutex.synchronize do
86
- next @seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)
104
+ next supported unless generation == @schema_generation
87
105
 
88
- rows = connection.exec_params(<<~SQL, [])
89
- SELECT 1 AS present FROM information_schema.columns
90
- WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
91
- AND table_schema = ANY (current_schemas(false))
92
- SQL
93
106
  @probed_at = monotonic_now
94
- @seats_supported = !rows.to_a.empty?
107
+ @seats_supported ||= supported
95
108
  end
96
109
  end
97
110
 
@@ -108,6 +121,7 @@ module Pgbus
108
121
 
109
122
  def reset_schema_cache!
110
123
  @schema_mutex.synchronize do
124
+ @schema_generation += 1
111
125
  @probed_at = nil
112
126
  @seats_supported = nil
113
127
  end
@@ -124,7 +138,7 @@ module Pgbus
124
138
  def expire_seats!(ttl_for:, min_ttl:)
125
139
  return [] unless seats_supported?
126
140
 
127
- candidates = connection.exec_params(<<~SQL, [min_ttl])
141
+ candidates = exec_params(<<~SQL, [min_ttl])
128
142
  SELECT stream_name, connection_id FROM pgbus_presence_members
129
143
  WHERE connection_id IS NOT NULL AND last_seen_at < NOW() - make_interval(secs => $1)
130
144
  SQL
@@ -133,23 +147,23 @@ module Pgbus
133
147
  end
134
148
  end
135
149
 
150
+ # The PG::Connection under the adapter. Run SQL through exec_params,
151
+ # not on this: the adapter's lock is released once it is returned.
136
152
  def connection
137
- # Respect multi-database setups (connects_to). When
138
- # Pgbus.configuration.connects_to is set, the pgbus tables
139
- # live in a separate database accessed via Pgbus::BusRecord;
140
- # the default path uses ActiveRecord::Base. Matches the
141
- # canonical pattern in Pgbus::Process::QueueLock#connection
142
- # and Pgbus::Configuration's connection probe.
143
- unless defined?(::ActiveRecord::Base)
144
- raise Pgbus::ConfigurationError,
145
- "Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
146
- end
153
+ adapter.raw_connection
154
+ end
147
155
 
148
- if Pgbus.configuration.connects_to
149
- Pgbus::BusRecord.connection.raw_connection
150
- else
151
- ::ActiveRecord::Base.connection.raw_connection
152
- end
156
+ # Runs one statement on the adapter's PG::Connection while holding
157
+ # the adapter's lock (issue #566). raw_connection takes that lock
158
+ # only to hand the connection out; on a pinned pool (transactional
159
+ # tests) every thread shares the one adapter, and its lock is all
160
+ # that keeps a second thread's statement off the libpq connection
161
+ # while this one is in flight. Unpinned, the lock is a NullLock. The
162
+ # lock is reentrant, so a thread already holding it (a request
163
+ # inside a transaction) re-enters it.
164
+ def exec_params(sql, binds)
165
+ adapter = self.adapter
166
+ adapter.lock.synchronize { adapter.raw_connection.exec_params(sql, binds) }
153
167
  end
154
168
 
155
169
  # Reports a seat join/leave to streams_presence_on_change (issue #515).
@@ -171,6 +185,21 @@ module Pgbus
171
185
 
172
186
  private
173
187
 
188
+ # Respect multi-database setups (connects_to). When
189
+ # Pgbus.configuration.connects_to is set, the pgbus tables
190
+ # live in a separate database accessed via Pgbus::BusRecord;
191
+ # the default path uses ActiveRecord::Base. Matches the
192
+ # canonical pattern in Pgbus::Process::QueueLock#connection
193
+ # and Pgbus::Configuration's connection probe.
194
+ def adapter
195
+ unless defined?(::ActiveRecord::Base)
196
+ raise Pgbus::ConfigurationError,
197
+ "Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
198
+ end
199
+
200
+ Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ::ActiveRecord::Base.connection
201
+ end
202
+
174
203
  def schema_answer_fresh?(supported, probed_at)
175
204
  return false if supported.nil? || probed_at.nil?
176
205
 
@@ -178,7 +207,7 @@ module Pgbus
178
207
  end
179
208
 
180
209
  def expire_stream_seats(stream_name, connection_ids, ttl)
181
- rows = connection.exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
210
+ rows = exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
182
211
  DELETE FROM pgbus_presence_members
183
212
  WHERE stream_name = $1 AND connection_id = ANY($2::text[])
184
213
  AND last_seen_at < NOW() - make_interval(secs => $3)
@@ -306,7 +335,7 @@ module Pgbus
306
335
  return touch_seats(Array(connection_id)) if connection_id
307
336
 
308
337
  member_id = member_id.to_s
309
- connection.exec_params(<<~SQL, [@stream.name, member_id])
338
+ exec_params(<<~SQL, [@stream.name, member_id])
310
339
  UPDATE pgbus_presence_members
311
340
  SET last_seen_at = NOW()
312
341
  WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
@@ -320,7 +349,7 @@ module Pgbus
320
349
  def set_state(connection_id:, state:)
321
350
  raise ArgumentError, "set_state takes a Hash (got #{state.class})" unless state.is_a?(Hash)
322
351
 
323
- rows = connection.exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
352
+ rows = exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
324
353
  UPDATE pgbus_presence_members
325
354
  SET state = state || $3::jsonb, last_seen_at = NOW()
326
355
  WHERE stream_name = $1 AND connection_id = $2
@@ -335,7 +364,7 @@ module Pgbus
335
364
  # table each row is one seat and also carries connection_id, seat
336
365
  # and state; group by "id" for one entry per member.
337
366
  def members
338
- rows = connection.exec_params(<<~SQL, [@stream.name])
367
+ rows = exec_params(<<~SQL, [@stream.name])
339
368
  SELECT #{seats? ? SEAT_RECORD_COLUMNS : RECORD_COLUMNS}
340
369
  FROM pgbus_presence_members
341
370
  WHERE stream_name = $1
@@ -347,7 +376,7 @@ module Pgbus
347
376
  # Returns the count of current members. Faster than members.size
348
377
  # because it doesn't deserialize metadata.
349
378
  def count
350
- rows = connection.exec_params(<<~SQL, [@stream.name])
379
+ rows = exec_params(<<~SQL, [@stream.name])
351
380
  SELECT COUNT(*) AS n FROM pgbus_presence_members WHERE stream_name = $1
352
381
  SQL
353
382
  rows.first["n"].to_i
@@ -362,7 +391,7 @@ module Pgbus
362
391
  # workers running the sweep concurrently won't double-emit leave
363
392
  # events. Yields each expired record when a block is given.
364
393
  def sweep!(older_than:)
365
- rows = connection.exec_params(<<~SQL, [@stream.name, older_than])
394
+ rows = exec_params(<<~SQL, [@stream.name, older_than])
366
395
  DELETE FROM pgbus_presence_members
367
396
  WHERE stream_name = $1 AND last_seen_at < $2#{" AND connection_id IS NULL" if seats_now?}
368
397
  RETURNING #{RECORD_COLUMNS}
@@ -388,7 +417,7 @@ module Pgbus
388
417
 
389
418
  def upsert_member_once(member_id, metadata)
390
419
  conflict = seats? ? "(stream_name, member_id) WHERE connection_id IS NULL" : "(stream_name, member_id)"
391
- rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
420
+ rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
392
421
  INSERT INTO pgbus_presence_members (stream_name, member_id, metadata, joined_at, last_seen_at)
393
422
  VALUES ($1, $2, $3::jsonb, NOW(), NOW())
394
423
  ON CONFLICT #{conflict}
@@ -399,7 +428,7 @@ module Pgbus
399
428
  end
400
429
 
401
430
  def upsert_seat(member_id, metadata, connection_id, seat)
402
- rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
431
+ rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
403
432
  INSERT INTO pgbus_presence_members
404
433
  (stream_name, member_id, metadata, connection_id, seat, joined_at, last_seen_at)
405
434
  VALUES ($1, $2, $3::jsonb, $4, $5, NOW(), NOW())
@@ -412,7 +441,7 @@ module Pgbus
412
441
  end
413
442
 
414
443
  def delete_member(member_id)
415
- rows = connection.exec_params(<<~SQL, [@stream.name, member_id])
444
+ rows = exec_params(<<~SQL, [@stream.name, member_id])
416
445
  DELETE FROM pgbus_presence_members
417
446
  WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
418
447
  RETURNING #{RECORD_COLUMNS}
@@ -422,7 +451,7 @@ module Pgbus
422
451
  end
423
452
 
424
453
  def delete_seat(connection_id)
425
- rows = connection.exec_params(<<~SQL, [@stream.name, connection_id])
454
+ rows = exec_params(<<~SQL, [@stream.name, connection_id])
426
455
  DELETE FROM pgbus_presence_members
427
456
  WHERE stream_name = $1 AND connection_id = $2
428
457
  RETURNING #{SEAT_RECORD_COLUMNS}
@@ -434,7 +463,7 @@ module Pgbus
434
463
  def touch_seats(connection_ids)
435
464
  return [] if connection_ids.empty?
436
465
 
437
- rows = connection.exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
466
+ rows = exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
438
467
  UPDATE pgbus_presence_members
439
468
  SET last_seen_at = NOW()
440
469
  WHERE stream_name = $1 AND connection_id = ANY($2::text[])
@@ -443,8 +472,8 @@ module Pgbus
443
472
  rows.to_a.map { |row| row["connection_id"] }
444
473
  end
445
474
 
446
- def connection
447
- self.class.connection
475
+ def exec_params(sql, binds)
476
+ self.class.exec_params(sql, binds)
448
477
  end
449
478
  end
450
479
  end