pgbus 0.19.4 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -0
  3. data/README.md +96 -7
  4. data/Rakefile +6 -1
  5. data/app/controllers/pgbus/dashboard_controller.rb +2 -0
  6. data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
  7. data/app/controllers/pgbus/workflows_controller.rb +197 -0
  8. data/app/frontend/pgbus/modules/charts.js +130 -4
  9. data/app/frontend/pgbus/modules/charts.min.js +1 -1
  10. data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
  11. data/app/frontend/pgbus/style.css +1 -1
  12. data/app/helpers/pgbus/button_helper.rb +4 -2
  13. data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
  14. data/app/helpers/pgbus/workflows_helper.rb +165 -0
  15. data/app/models/pgbus/workflow_entry.rb +172 -0
  16. data/app/models/pgbus/workflow_event.rb +43 -0
  17. data/app/models/pgbus/workflow_signal.rb +77 -0
  18. data/app/models/pgbus/workflow_step.rb +142 -0
  19. data/app/views/layouts/pgbus/application.html.erb +2 -0
  20. data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
  21. data/app/views/pgbus/workflows/_actions.html.erb +59 -0
  22. data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
  23. data/app/views/pgbus/workflows/_filters.html.erb +29 -0
  24. data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
  25. data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
  26. data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
  27. data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
  28. data/app/views/pgbus/workflows/_timeline.html.erb +39 -0
  29. data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
  30. data/app/views/pgbus/workflows/attention.html.erb +88 -0
  31. data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
  32. data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
  33. data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
  34. data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
  35. data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
  36. data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
  37. data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
  38. data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
  39. data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
  40. data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
  41. data/app/views/pgbus/workflows/index.html.erb +15 -0
  42. data/app/views/pgbus/workflows/not_found.html.erb +5 -0
  43. data/app/views/pgbus/workflows/show.html.erb +93 -0
  44. data/config/i18n-tasks.yml +6 -0
  45. data/config/locales/da.yml +442 -0
  46. data/config/locales/de.yml +442 -0
  47. data/config/locales/en.yml +442 -0
  48. data/config/locales/es.yml +442 -0
  49. data/config/locales/fi.yml +442 -0
  50. data/config/locales/fr.yml +442 -0
  51. data/config/locales/it.yml +442 -0
  52. data/config/locales/ja.yml +442 -0
  53. data/config/locales/nb.yml +442 -0
  54. data/config/locales/nl.yml +442 -0
  55. data/config/locales/pt.yml +442 -0
  56. data/config/locales/sv.yml +442 -0
  57. data/config/routes.rb +26 -0
  58. data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
  59. data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
  60. data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
  61. data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
  62. data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
  63. data/lib/pgbus/active_job/delivery.rb +49 -0
  64. data/lib/pgbus/active_job/executor.rb +3 -1
  65. data/lib/pgbus/configuration/workflow_settings.rb +99 -0
  66. data/lib/pgbus/configuration.rb +21 -3
  67. data/lib/pgbus/generators/migration_detector.rb +40 -0
  68. data/lib/pgbus/instrumentation.rb +32 -0
  69. data/lib/pgbus/integrations/appsignal/dashboard.json +128 -1
  70. data/lib/pgbus/integrations/appsignal/subscriber.rb +16 -1
  71. data/lib/pgbus/mcp/rack_app.rb +7 -2
  72. data/lib/pgbus/mcp/runner.rb +2 -1
  73. data/lib/pgbus/mcp/serialization.rb +25 -0
  74. data/lib/pgbus/mcp/server.rb +53 -10
  75. data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
  76. data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
  77. data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
  78. data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
  79. data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
  80. data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
  81. data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
  82. data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
  83. data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
  84. data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
  85. data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
  86. data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
  87. data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
  88. data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
  89. data/lib/pgbus/mcp/write_tool.rb +134 -0
  90. data/lib/pgbus/mcp.rb +15 -0
  91. data/lib/pgbus/metrics/backends/statsd.rb +6 -0
  92. data/lib/pgbus/metrics/subscriber.rb +14 -1
  93. data/lib/pgbus/metrics/workflow_metrics.rb +65 -0
  94. data/lib/pgbus/process/dispatcher.rb +26 -0
  95. data/lib/pgbus/version.rb +1 -1
  96. data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
  97. data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
  98. data/lib/pgbus/web/data_source/workflow_list.rb +248 -0
  99. data/lib/pgbus/web/data_source.rb +3 -0
  100. data/lib/pgbus/web/workflow_action_plan.rb +62 -0
  101. data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
  102. data/lib/pgbus/web/workflow_insights_report.rb +171 -0
  103. data/lib/pgbus/web/workflow_payload.rb +36 -0
  104. data/lib/pgbus/web/workflow_query.rb +82 -0
  105. data/lib/pgbus/web/workflow_state.rb +76 -0
  106. data/lib/pgbus/web/workflow_timeline.rb +113 -0
  107. data/lib/pgbus/web/workflow_window.rb +51 -0
  108. data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
  109. data/lib/pgbus/workflow/dsl.rb +105 -0
  110. data/lib/pgbus/workflow/error_capture.rb +63 -0
  111. data/lib/pgbus/workflow/exception_policy.rb +170 -0
  112. data/lib/pgbus/workflow/fan_out.rb +157 -0
  113. data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
  114. data/lib/pgbus/workflow/flow_control.rb +94 -0
  115. data/lib/pgbus/workflow/instance.rb +183 -0
  116. data/lib/pgbus/workflow/outside_verbs.rb +138 -0
  117. data/lib/pgbus/workflow/payload.rb +53 -0
  118. data/lib/pgbus/workflow/registry.rb +53 -0
  119. data/lib/pgbus/workflow/removed_step.rb +14 -0
  120. data/lib/pgbus/workflow/runner.rb +218 -0
  121. data/lib/pgbus/workflow/signal_matcher.rb +53 -0
  122. data/lib/pgbus/workflow/signals.rb +132 -0
  123. data/lib/pgbus/workflow/states.rb +119 -0
  124. data/lib/pgbus/workflow/step_context.rb +58 -0
  125. data/lib/pgbus/workflow/step_definition.rb +131 -0
  126. data/lib/pgbus/workflow/step_execution.rb +242 -0
  127. data/lib/pgbus/workflow/step_job.rb +28 -0
  128. data/lib/pgbus/workflow/step_outcome.rb +151 -0
  129. data/lib/pgbus/workflow/steps.rb +103 -0
  130. data/lib/pgbus/workflow/sweep.rb +156 -0
  131. data/lib/pgbus/workflow/test_helpers.rb +152 -0
  132. data/lib/pgbus/workflow/wait_timeout.rb +73 -0
  133. data/lib/pgbus/workflow/waiting.rb +82 -0
  134. data/lib/pgbus/workflow.rb +183 -0
  135. data/sig/pgbus/configuration.rbs +22 -0
  136. metadata +92 -1
@@ -0,0 +1,150 @@
1
+ class AddPgbusWorkflows < ActiveRecord::Migration<%= migration_version %>
2
+ # if_not_exists throughout: pgbus:update queues this migration when any of
3
+ # the three core tables is missing, so a half-present schema must not fail
4
+ # it. The waiting-step columns and pgbus_workflow_signals (issue #541) are
5
+ # here too, so a fresh install needs no pgbus:add_workflow_signals.
6
+ # up/down rather than change: Rails 7.1 cannot reverse an add_foreign_key
7
+ # that names its primary_key, and dropping the tables drops the keys.
8
+ def up
9
+ # One row per workflow run (Pgbus::WorkflowEntry).
10
+ create_table :pgbus_workflows, if_not_exists: true do |t|
11
+ t.string :workflow_id, null: false
12
+ t.string :workflow_class, null: false
13
+ t.string :subject_type
14
+ t.string :subject_id
15
+ t.string :subject_gid
16
+ t.jsonb :arguments, null: false, default: {}
17
+ t.string :status, null: false, default: "ready"
18
+ t.string :current_step
19
+ t.jsonb :step_job_options, null: false, default: {}
20
+ t.string :error_class
21
+ t.text :error_message
22
+ t.text :error_backtrace
23
+ t.datetime :paused_at
24
+ t.datetime :finished_at
25
+ t.datetime :cancelled_at
26
+ t.string :idempotency_key
27
+ t.string :definition_digest, null: false
28
+ t.string :rerun_of_workflow_id
29
+ t.datetime :created_at, null: false
30
+ t.datetime :updated_at, null: false
31
+ end
32
+
33
+ add_index :pgbus_workflows, :workflow_id,
34
+ unique: true, name: "idx_pgbus_workflows_workflow_id", if_not_exists: true
35
+ add_index :pgbus_workflows, [:workflow_class, :subject_type, :subject_id],
36
+ unique: true, where: "status IN ('ready','performing','waiting','paused')",
37
+ name: "idx_pgbus_workflows_one_ongoing", if_not_exists: true
38
+ add_index :pgbus_workflows, [:workflow_class, :idempotency_key],
39
+ unique: true, where: "idempotency_key IS NOT NULL",
40
+ name: "idx_pgbus_workflows_idempotency", if_not_exists: true
41
+ add_index :pgbus_workflows, [:status, :updated_at],
42
+ name: "idx_pgbus_workflows_status", if_not_exists: true
43
+ add_index :pgbus_workflows, [:workflow_class, :status, :created_at],
44
+ name: "idx_pgbus_workflows_class_status", if_not_exists: true
45
+ add_index :pgbus_workflows, [:subject_type, :subject_id],
46
+ name: "idx_pgbus_workflows_subject", if_not_exists: true
47
+
48
+ # One row per step attempt (Pgbus::WorkflowStep).
49
+ create_table :pgbus_workflow_steps, if_not_exists: true do |t|
50
+ t.string :workflow_id, null: false
51
+ t.string :workflow_class, null: false
52
+ t.string :step_name, null: false
53
+ t.integer :attempt, null: false, default: 1
54
+ t.string :status, null: false, default: "scheduled"
55
+ t.string :job_id
56
+ t.bigint :msg_id
57
+ t.datetime :scheduled_at, null: false
58
+ t.datetime :enqueued_at
59
+ t.datetime :started_at
60
+ t.datetime :finished_at
61
+ t.datetime :stall_at
62
+ t.string :outcome
63
+ t.jsonb :output
64
+ t.string :error_class
65
+ t.text :error_message
66
+ t.text :error_backtrace
67
+ t.jsonb :error_causes
68
+ t.string :worker
69
+ t.string :app_revision
70
+ # Waiting steps (fan_out, wait_for:): also added by pgbus:add_workflow_signals.
71
+ t.string :batch_id
72
+ t.string :wait_for
73
+ t.bigint :signal_id
74
+ t.datetime :waiting_since
75
+ t.datetime :timeout_at
76
+ t.datetime :created_at, null: false
77
+ end
78
+
79
+ add_index :pgbus_workflow_steps, [:workflow_id, :step_name, :attempt],
80
+ unique: true, name: "idx_pgbus_workflow_steps_attempt", if_not_exists: true
81
+ add_index :pgbus_workflow_steps, [:status, :scheduled_at],
82
+ name: "idx_pgbus_workflow_steps_scheduled", if_not_exists: true
83
+ add_index :pgbus_workflow_steps, [:status, :stall_at],
84
+ name: "idx_pgbus_workflow_steps_stall", if_not_exists: true
85
+ add_index :pgbus_workflow_steps, [:workflow_class, :step_name, :finished_at],
86
+ name: "idx_pgbus_workflow_steps_insights", if_not_exists: true
87
+ # A steps table that predates waiting steps lacks these columns (the
88
+ # queued pgbus:add_workflow_signals adds them and their indexes).
89
+ if column_exists?(:pgbus_workflow_steps, :batch_id)
90
+ add_index :pgbus_workflow_steps, :batch_id,
91
+ name: "idx_pgbus_workflow_steps_batch", if_not_exists: true
92
+ end
93
+ if column_exists?(:pgbus_workflow_steps, :timeout_at)
94
+ add_index :pgbus_workflow_steps, [:status, :timeout_at],
95
+ name: "idx_pgbus_workflow_steps_timeout", if_not_exists: true
96
+ end
97
+ add_foreign_key :pgbus_workflow_steps, :pgbus_workflows,
98
+ column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade, if_not_exists: true
99
+
100
+ # Append-only audit trail: one row per transition or operator action
101
+ # (Pgbus::WorkflowEvent).
102
+ create_table :pgbus_workflow_events, if_not_exists: true do |t|
103
+ t.string :workflow_id, null: false
104
+ t.bigint :step_id
105
+ t.string :step_name
106
+ t.integer :attempt
107
+ t.string :event, null: false
108
+ t.string :actor, null: false
109
+ t.text :reason
110
+ t.jsonb :details, null: false, default: {}
111
+ t.datetime :created_at, null: false
112
+ end
113
+
114
+ add_index :pgbus_workflow_events, [:workflow_id, :id],
115
+ name: "idx_pgbus_workflow_events_workflow", if_not_exists: true
116
+ add_foreign_key :pgbus_workflow_events, :pgbus_workflows,
117
+ column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade, if_not_exists: true
118
+
119
+ # One row per signal delivered to a run (Pgbus::WorkflowSignal):
120
+ # pending → claimed → consumed.
121
+ create_table :pgbus_workflow_signals, if_not_exists: true do |t|
122
+ t.string :workflow_id, null: false
123
+ t.string :name, null: false
124
+ t.jsonb :payload, null: false, default: {}
125
+ t.string :idempotency_key
126
+ t.string :status, null: false, default: "pending"
127
+ t.bigint :claimed_by_step_id
128
+ t.datetime :created_at, null: false
129
+ t.datetime :claimed_at
130
+ t.datetime :consumed_at
131
+ end
132
+
133
+ add_index :pgbus_workflow_signals, [:workflow_id, :idempotency_key],
134
+ unique: true, where: "idempotency_key IS NOT NULL",
135
+ name: "idx_pgbus_workflow_signals_idempotency", if_not_exists: true
136
+ add_index :pgbus_workflow_signals, [:workflow_id, :name, :status],
137
+ name: "idx_pgbus_workflow_signals_pending", if_not_exists: true
138
+ add_index :pgbus_workflow_signals, [:workflow_id, :status, :claimed_by_step_id],
139
+ name: "idx_pgbus_workflow_signals_claimed", if_not_exists: true
140
+ add_foreign_key :pgbus_workflow_signals, :pgbus_workflows,
141
+ column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade, if_not_exists: true
142
+ end
143
+
144
+ def down
145
+ drop_table :pgbus_workflow_signals, if_exists: true
146
+ drop_table :pgbus_workflow_events, if_exists: true
147
+ drop_table :pgbus_workflow_steps, if_exists: true
148
+ drop_table :pgbus_workflows, if_exists: true
149
+ end
150
+ end
@@ -211,6 +211,129 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
211
211
  add_index :pgbus_deploy_task_runs, :run_at,
212
212
  name: "idx_pgbus_deploy_task_runs_cleanup"
213
213
 
214
+ # Workflows (Pgbus::Workflow): one row per run, one per step attempt, and
215
+ # the append-only audit trail. At most one ongoing run per (class, subject).
216
+ create_table :pgbus_workflows do |t|
217
+ t.string :workflow_id, null: false
218
+ t.string :workflow_class, null: false
219
+ t.string :subject_type
220
+ t.string :subject_id
221
+ t.string :subject_gid
222
+ t.jsonb :arguments, null: false, default: {}
223
+ t.string :status, null: false, default: "ready"
224
+ t.string :current_step
225
+ t.jsonb :step_job_options, null: false, default: {}
226
+ t.string :error_class
227
+ t.text :error_message
228
+ t.text :error_backtrace
229
+ t.datetime :paused_at
230
+ t.datetime :finished_at
231
+ t.datetime :cancelled_at
232
+ t.string :idempotency_key
233
+ t.string :definition_digest, null: false
234
+ t.string :rerun_of_workflow_id
235
+ t.datetime :created_at, null: false
236
+ t.datetime :updated_at, null: false
237
+ end
238
+
239
+ add_index :pgbus_workflows, :workflow_id,
240
+ unique: true, name: "idx_pgbus_workflows_workflow_id"
241
+ add_index :pgbus_workflows, [:workflow_class, :subject_type, :subject_id],
242
+ unique: true, where: "status IN ('ready','performing','waiting','paused')",
243
+ name: "idx_pgbus_workflows_one_ongoing"
244
+ add_index :pgbus_workflows, [:workflow_class, :idempotency_key],
245
+ unique: true, where: "idempotency_key IS NOT NULL",
246
+ name: "idx_pgbus_workflows_idempotency"
247
+ add_index :pgbus_workflows, [:status, :updated_at],
248
+ name: "idx_pgbus_workflows_status"
249
+ add_index :pgbus_workflows, [:workflow_class, :status, :created_at],
250
+ name: "idx_pgbus_workflows_class_status"
251
+ add_index :pgbus_workflows, [:subject_type, :subject_id],
252
+ name: "idx_pgbus_workflows_subject"
253
+
254
+ create_table :pgbus_workflow_steps do |t|
255
+ t.string :workflow_id, null: false
256
+ t.string :workflow_class, null: false
257
+ t.string :step_name, null: false
258
+ t.integer :attempt, null: false, default: 1
259
+ t.string :status, null: false, default: "scheduled"
260
+ t.string :job_id
261
+ t.bigint :msg_id
262
+ t.datetime :scheduled_at, null: false
263
+ t.datetime :enqueued_at
264
+ t.datetime :started_at
265
+ t.datetime :finished_at
266
+ t.datetime :stall_at
267
+ t.string :outcome
268
+ t.jsonb :output
269
+ t.string :error_class
270
+ t.text :error_message
271
+ t.text :error_backtrace
272
+ t.jsonb :error_causes
273
+ t.string :worker
274
+ t.string :app_revision
275
+ t.string :batch_id
276
+ t.string :wait_for
277
+ t.bigint :signal_id
278
+ t.datetime :waiting_since
279
+ t.datetime :timeout_at
280
+ t.datetime :created_at, null: false
281
+ end
282
+
283
+ add_index :pgbus_workflow_steps, [:workflow_id, :step_name, :attempt],
284
+ unique: true, name: "idx_pgbus_workflow_steps_attempt"
285
+ add_index :pgbus_workflow_steps, [:status, :scheduled_at],
286
+ name: "idx_pgbus_workflow_steps_scheduled"
287
+ add_index :pgbus_workflow_steps, [:status, :stall_at],
288
+ name: "idx_pgbus_workflow_steps_stall"
289
+ add_index :pgbus_workflow_steps, [:workflow_class, :step_name, :finished_at],
290
+ name: "idx_pgbus_workflow_steps_insights"
291
+ add_index :pgbus_workflow_steps, :batch_id,
292
+ name: "idx_pgbus_workflow_steps_batch"
293
+ add_index :pgbus_workflow_steps, [:status, :timeout_at],
294
+ name: "idx_pgbus_workflow_steps_timeout"
295
+ add_foreign_key :pgbus_workflow_steps, :pgbus_workflows,
296
+ column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade
297
+
298
+ create_table :pgbus_workflow_events do |t|
299
+ t.string :workflow_id, null: false
300
+ t.bigint :step_id
301
+ t.string :step_name
302
+ t.integer :attempt
303
+ t.string :event, null: false
304
+ t.string :actor, null: false
305
+ t.text :reason
306
+ t.jsonb :details, null: false, default: {}
307
+ t.datetime :created_at, null: false
308
+ end
309
+
310
+ add_index :pgbus_workflow_events, [:workflow_id, :id],
311
+ name: "idx_pgbus_workflow_events_workflow"
312
+ add_foreign_key :pgbus_workflow_events, :pgbus_workflows,
313
+ column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade
314
+
315
+ create_table :pgbus_workflow_signals do |t|
316
+ t.string :workflow_id, null: false
317
+ t.string :name, null: false
318
+ t.jsonb :payload, null: false, default: {}
319
+ t.string :idempotency_key
320
+ t.string :status, null: false, default: "pending"
321
+ t.bigint :claimed_by_step_id
322
+ t.datetime :created_at, null: false
323
+ t.datetime :claimed_at
324
+ t.datetime :consumed_at
325
+ end
326
+
327
+ add_index :pgbus_workflow_signals, [:workflow_id, :idempotency_key],
328
+ unique: true, where: "idempotency_key IS NOT NULL",
329
+ name: "idx_pgbus_workflow_signals_idempotency"
330
+ add_index :pgbus_workflow_signals, [:workflow_id, :name, :status],
331
+ name: "idx_pgbus_workflow_signals_pending"
332
+ add_index :pgbus_workflow_signals, [:workflow_id, :status, :claimed_by_step_id],
333
+ name: "idx_pgbus_workflow_signals_claimed"
334
+ add_foreign_key :pgbus_workflow_signals, :pgbus_workflows,
335
+ column: :workflow_id, primary_key: :workflow_id, on_delete: :cascade
336
+
214
337
  # Stream queue registry — records which PGMQ queues back streams (vs jobs).
215
338
  # Stream and job queues share the pgbus_ namespace, so maintenance
216
339
  # (per-stream retention, orphan sweep) and wildcard workers rely on this
@@ -245,6 +368,10 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
245
368
  execute "SELECT pgmq.drop_queue('pgbus_default')"
246
369
 
247
370
  drop_table :pgbus_stream_queues
371
+ drop_table :pgbus_workflow_signals
372
+ drop_table :pgbus_workflow_events
373
+ drop_table :pgbus_workflow_steps
374
+ drop_table :pgbus_workflows
248
375
  drop_table :pgbus_deploy_task_runs
249
376
  drop_table :pgbus_deploy_tasks
250
377
  drop_table :pgbus_recurring_executions
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module ActiveJob
5
+ # The PGMQ delivery of the job running in this fiber:
6
+ #
7
+ # Pgbus::ActiveJob::Delivery.current # => { msg_id: 42, read_ct: 2, queue_name: "default" }
8
+ #
9
+ # Set by Executor#execute around perform_now and cleared after, also when
10
+ # the job raises; nil outside the executor (perform_now in a test, a
11
+ # top-level inline-adapter job). A job run inline from inside an executing
12
+ # job sees the enclosing job's delivery; StepJob only acts on it when the
13
+ # step row's msg_id matches, so that is harmless. Fiber-local (Thread.current[]), so concurrent jobs under
14
+ # execution_mode: :async each see their own. Pgbus::Workflow::StepJob reads
15
+ # read_ct to tell a redelivery after a worker died from a first delivery.
16
+ #
17
+ # The three values sit in three fiber-locals and the Hash is built only
18
+ # when read: setting them allocates nothing, which keeps the executor's
19
+ # per-job allocation budget (spec/pgbus/active_job/executor_allocation_budget_spec.rb).
20
+ module Delivery
21
+ MSG_ID_KEY = :pgbus_delivery_msg_id
22
+ READ_CT_KEY = :pgbus_delivery_read_ct
23
+ QUEUE_NAME_KEY = :pgbus_delivery_queue_name
24
+
25
+ module_function
26
+
27
+ def current
28
+ msg_id = Thread.current[MSG_ID_KEY]
29
+ return nil if msg_id.nil?
30
+
31
+ { msg_id: msg_id, read_ct: Thread.current[READ_CT_KEY], queue_name: Thread.current[QUEUE_NAME_KEY] }
32
+ end
33
+
34
+ def with(msg_id:, read_ct:, queue_name:)
35
+ previous_msg_id = Thread.current[MSG_ID_KEY]
36
+ previous_read_ct = Thread.current[READ_CT_KEY]
37
+ previous_queue_name = Thread.current[QUEUE_NAME_KEY]
38
+ Thread.current[MSG_ID_KEY] = msg_id
39
+ Thread.current[READ_CT_KEY] = read_ct
40
+ Thread.current[QUEUE_NAME_KEY] = queue_name
41
+ yield
42
+ ensure
43
+ Thread.current[MSG_ID_KEY] = previous_msg_id
44
+ Thread.current[READ_CT_KEY] = previous_read_ct
45
+ Thread.current[QUEUE_NAME_KEY] = previous_queue_name
46
+ end
47
+ end
48
+ end
49
+ end
@@ -99,7 +99,9 @@ module Pgbus
99
99
  # `batch` (and `batch.enqueue` for open batches) work inside a job.
100
100
  assign_batch_id(job, payload)
101
101
  Pgbus.logger.debug { "[Pgbus::Executor] running #{log_tag(message, queue_name)} job_class=#{job_class}" }
102
- with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) { execute_job(job) }
102
+ with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) do
103
+ Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name) { execute_job(job) }
104
+ end
103
105
  # retry_on re-enqueues from inside perform_now and returns normally:
104
106
  # this attempt is done (archive it) but the job is not — the retry
105
107
  # message carries the batch tag and signals on its own outcome.
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ class Configuration
5
+ # Workflows (issue #540): durable multi-step Pgbus::Workflow runs.
6
+ #
7
+ # config.workflow_queue = "workflows" # where StepJob messages go
8
+ # config.workflow_retention = 30.days # finished/cancelled runs kept this long; nil keeps them
9
+ # config.workflow_sweep_interval = 5.minutes # how often the dispatcher repairs stuck steps
10
+ # config.workflow_stall_threshold = 1.hour # an in_progress step older than this is stalled
11
+ # config.workflow_stall_action = :pause # :pause, :reattempt or :cancel a stalled step
12
+ # config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
13
+ # config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
14
+ # config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
15
+ # # wait_timeout: counts as stuck (waiting_overdue)
16
+ # config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
17
+ # # the dashboard's subject link (nil: plain text)
18
+ module WorkflowSettings
19
+ WORKFLOW_STALL_ACTIONS = %i[pause reattempt cancel].freeze
20
+
21
+ attr_reader :workflow_queue, :workflow_retention, :workflow_sweep_interval, :workflow_stall_threshold,
22
+ :workflow_stall_action, :workflow_waiting_visibility_threshold
23
+ attr_accessor :workflow_max_payload_size, :workflow_error_backtrace_lines, :workflow_subject_url
24
+
25
+ def workflow_queue=(value)
26
+ name = value.to_s
27
+ raise Pgbus::ConfigurationError, "workflow_queue must be a non-empty queue name" if value.nil? || name.strip.empty?
28
+
29
+ @workflow_queue = name
30
+ end
31
+
32
+ def workflow_retention=(value)
33
+ @workflow_retention = coerce_duration!(value, :workflow_retention)
34
+ end
35
+
36
+ def workflow_sweep_interval=(value)
37
+ @workflow_sweep_interval = coerce_duration!(value, :workflow_sweep_interval)
38
+ end
39
+
40
+ def workflow_stall_threshold=(value)
41
+ @workflow_stall_threshold = coerce_duration!(value, :workflow_stall_threshold)
42
+ end
43
+
44
+ def workflow_waiting_visibility_threshold=(value)
45
+ @workflow_waiting_visibility_threshold = coerce_duration!(value, :workflow_waiting_visibility_threshold)
46
+ end
47
+
48
+ def workflow_stall_action=(value)
49
+ action = value.respond_to?(:to_sym) ? value.to_sym : value
50
+ unless WORKFLOW_STALL_ACTIONS.include?(action)
51
+ raise Pgbus::ConfigurationError,
52
+ "workflow_stall_action must be one of #{WORKFLOW_STALL_ACTIONS.map(&:inspect).join(", ")}, got #{value.inspect}"
53
+ end
54
+
55
+ @workflow_stall_action = action
56
+ end
57
+
58
+ private
59
+
60
+ def initialize_workflow_settings
61
+ @workflow_queue = "default"
62
+ @workflow_retention = 30 * 24 * 3600 # 30 days
63
+ @workflow_sweep_interval = 300 # 5 minutes
64
+ @workflow_stall_threshold = 3600 # 1 hour
65
+ @workflow_stall_action = :pause
66
+ @workflow_max_payload_size = 131_072 # 128 KB
67
+ @workflow_error_backtrace_lines = 50
68
+ @workflow_waiting_visibility_threshold = 7 * 24 * 3600 # 7 days
69
+ @workflow_subject_url = nil
70
+ end
71
+
72
+ def validate_workflow_settings!
73
+ { workflow_sweep_interval: workflow_sweep_interval, workflow_stall_threshold: workflow_stall_threshold,
74
+ workflow_waiting_visibility_threshold: workflow_waiting_visibility_threshold,
75
+ workflow_retention: workflow_retention }.each do |name, value|
76
+ next if value.nil? && name == :workflow_retention
77
+ next if value.is_a?(Numeric) && value.positive? && value.finite?
78
+
79
+ raise Pgbus::ConfigurationError, "#{name} must be a finite number of seconds > 0"
80
+ end
81
+
82
+ begin
83
+ QueueNameValidator.validate!(queue_name(workflow_queue))
84
+ rescue ArgumentError => e
85
+ raise Pgbus::ConfigurationError, "workflow_queue #{workflow_queue.inspect}: #{e.message}"
86
+ end
87
+
88
+ { workflow_max_payload_size: workflow_max_payload_size,
89
+ workflow_error_backtrace_lines: workflow_error_backtrace_lines }.each do |name, value|
90
+ raise Pgbus::ConfigurationError, "#{name} must be a positive integer" unless value.is_a?(Integer) && value.positive?
91
+ end
92
+
93
+ return if workflow_subject_url.nil? || workflow_subject_url.respond_to?(:call)
94
+
95
+ raise Pgbus::ConfigurationError, "workflow_subject_url must respond to #call (a Proc/lambda) or be nil"
96
+ end
97
+ end
98
+ end
99
+ end
@@ -7,6 +7,7 @@ module Pgbus
7
7
  include PresenceSettings
8
8
  include StreamAuthorization
9
9
  include DeploySettings
10
+ include WorkflowSettings
10
11
 
11
12
  # Connection settings
12
13
  attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
@@ -215,10 +216,16 @@ module Pgbus
215
216
 
216
217
  # Web dashboard
217
218
  attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
218
- :insights_default_minutes, :base_controller_class, :return_to_app_url,
219
+ :insights_default_minutes, :base_controller_class, :return_to_app_url, :web_actor,
219
220
  :metrics_enabled,
220
221
  :web_filter_parameters, :web_filter_sensitive
221
222
 
223
+ # The MCP server's workflow write tools (issue #550): pgbus_workflow_resume,
224
+ # _cancel, _skip_step, _resume_at, _run_now and _signal. Off by default, and
225
+ # then they are never built, so tools/list cannot show them. Read once, when
226
+ # Pgbus::MCP::Server.build runs (pgbus mcp, Pgbus::MCP.rack_app).
227
+ attr_accessor :mcp_write_tools
228
+
222
229
  # HTTP health endpoints (liveness/readiness for orchestrators like
223
230
  # Kubernetes). `health_port` nil (default) leaves the standalone server in
224
231
  # the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
@@ -365,6 +372,7 @@ module Pgbus
365
372
  @recurring_enabled = true
366
373
  @recurring_execution_retention = 7 * 24 * 3600 # 7 days
367
374
  initialize_deploy_settings
375
+ initialize_workflow_settings
368
376
 
369
377
  @zombie_detection = true
370
378
 
@@ -380,7 +388,13 @@ module Pgbus
380
388
  @require_primary = false
381
389
  @connection_guc_mode = :options
382
390
 
391
+ @mcp_write_tools = false
392
+
383
393
  @web_auth = nil
394
+ # ->(request) { String or nil }: who an operator action on the dashboard
395
+ # is recorded as ("dashboard:<name>"; nil → "dashboard"). web_auth only
396
+ # answers allow/deny, so the name comes from here.
397
+ @web_actor = nil
384
398
  @web_refresh_interval = 5000
385
399
  @web_per_page = 25
386
400
  @web_live_updates = true
@@ -941,12 +955,14 @@ module Pgbus
941
955
  end
942
956
 
943
957
  raise Pgbus::ConfigurationError, "require_primary must be true or false" unless [true, false].include?(require_primary)
958
+ raise Pgbus::ConfigurationError, "mcp_write_tools must be true or false" unless [true, false].include?(mcp_write_tools)
944
959
 
945
960
  validate_job_path_gaps!
946
961
  validate_streams!
947
962
  validate_metrics_backend!
948
963
  validate_fair_share!
949
964
  validate_deploy_settings!
965
+ validate_workflow_settings!
950
966
 
951
967
  self
952
968
  end
@@ -1062,8 +1078,10 @@ module Pgbus
1062
1078
 
1063
1079
  # web_auth gates the dashboard; a non-callable value silently leaves it
1064
1080
  # open (mirror streams_presence_member).
1065
- unless web_auth.nil? || web_auth.respond_to?(:call)
1066
- raise Pgbus::ConfigurationError, "web_auth must respond to #call (a Proc/lambda) or be nil"
1081
+ { web_auth: web_auth, web_actor: web_actor }.each do |name, value|
1082
+ next if value.nil? || value.respond_to?(:call)
1083
+
1084
+ raise Pgbus::ConfigurationError, "#{name} must respond to #call (a Proc/lambda) or be nil"
1067
1085
  end
1068
1086
 
1069
1087
  raise Pgbus::ConfigurationError, "error_reporters must be an Array" unless error_reporters.is_a?(Array)
@@ -57,6 +57,22 @@ module Pgbus
57
57
  pgbus_batches
58
58
  ].freeze
59
59
 
60
+ # The three tables pgbus:add_workflows creates (issue #540).
61
+ WORKFLOW_TABLES = %w[pgbus_workflows pgbus_workflow_steps pgbus_workflow_events].freeze
62
+
63
+ # The waiting-step columns pgbus:add_workflow_signals adds to
64
+ # pgbus_workflow_steps (issue #541), with the pgbus_workflow_signals table.
65
+ WORKFLOW_WAITING_COLUMNS = %w[batch_id wait_for signal_id waiting_since timeout_at].freeze
66
+ WORKFLOW_WAITING_INDEXES = Ractor.make_shareable(
67
+ [
68
+ %w[pgbus_workflow_steps idx_pgbus_workflow_steps_batch],
69
+ %w[pgbus_workflow_steps idx_pgbus_workflow_steps_timeout],
70
+ %w[pgbus_workflow_signals idx_pgbus_workflow_signals_idempotency],
71
+ %w[pgbus_workflow_signals idx_pgbus_workflow_signals_pending],
72
+ %w[pgbus_workflow_signals idx_pgbus_workflow_signals_claimed]
73
+ ]
74
+ )
75
+
60
76
  # generator_key → Rails generator name. Passed to Thor's invoke.
61
77
  #
62
78
  # Note: uniqueness_keys invokes add_uniqueness_keys, which only
@@ -78,6 +94,8 @@ module Pgbus
78
94
  add_outbox: "pgbus:add_outbox",
79
95
  add_recurring: "pgbus:add_recurring",
80
96
  add_deploy_tasks: "pgbus:add_deploy_tasks",
97
+ add_workflows: "pgbus:add_workflows",
98
+ add_workflow_signals: "pgbus:add_workflow_signals",
81
99
  add_failed_events_index: "pgbus:add_failed_events_index",
82
100
  add_processed_event_completion: "pgbus:add_processed_event_completion",
83
101
  add_batch_executions: "pgbus:add_batch_executions",
@@ -101,6 +119,8 @@ module Pgbus
101
119
  add_outbox: "outbox entries table (transactional outbox)",
102
120
  add_recurring: "recurring tasks + executions tables",
103
121
  add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
122
+ add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
123
+ add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
104
124
  add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
105
125
  add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
106
126
  add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
@@ -130,6 +150,7 @@ module Pgbus
130
150
  *outbox_migrations,
131
151
  *recurring_migrations,
132
152
  *deploy_tasks_migrations,
153
+ *workflows_migrations,
133
154
  *failed_events_index_migrations,
134
155
  *processed_event_completion_migrations,
135
156
  *batch_executions_migrations,
@@ -226,6 +247,25 @@ module Pgbus
226
247
  [:add_deploy_tasks]
227
248
  end
228
249
 
250
+ # The add_workflows template uses if_not_exists throughout, so a
251
+ # half-present set of the three tables is safe to queue again.
252
+ #
253
+ # add_workflows carries the waiting-step schema too, so a database with
254
+ # no steps table needs only that one; a steps table without the columns
255
+ # (or a missing signals table) predates waiting steps.
256
+ def workflows_migrations
257
+ migrations = []
258
+ migrations << :add_workflows unless WORKFLOW_TABLES.all? { |table| table_exists?(table) }
259
+ return migrations unless table_exists?("pgbus_workflow_steps")
260
+
261
+ columns = column_names("pgbus_workflow_steps")
262
+ waiting = WORKFLOW_WAITING_COLUMNS.all? { |column| columns.include?(column) } &&
263
+ table_exists?("pgbus_workflow_signals") &&
264
+ WORKFLOW_WAITING_INDEXES.all? { |table, index| index_exists?(table, index) }
265
+ migrations << :add_workflow_signals unless waiting
266
+ migrations
267
+ end
268
+
229
269
  def failed_events_index_migrations
230
270
  # pgbus_failed_events is created by pgbus:install, but the
231
271
  # unique (queue_name, msg_id) index was added later via its own
@@ -54,6 +54,38 @@ module Pgbus
54
54
  # pgbus.batch_sweep — dispatcher stalled-batch sweep
55
55
  # payload: stalled_for, stale_executions, orphan_rows,
56
56
  # started_batches, finished_batches
57
+ # pgbus.workflow.step — a workflow step attempt ended (body, guard, stall)
58
+ # payload: workflow_class, workflow_id, step_name, attempt,
59
+ # outcome (completed | reattempted | skipped | paused |
60
+ # cancelled | finished_early), status (the step row's),
61
+ # error_class (nil without an error), duration (seconds
62
+ # the body ran; 0.0 for a guard; nil for a stall),
63
+ # pickup_latency (seconds from scheduled_at to the claim)
64
+ # pgbus.workflow_finished — a workflow run finished
65
+ # payload: workflow_class, workflow_id
66
+ # pgbus.workflow_paused — a workflow run paused (an exception, a stall, pause!)
67
+ # payload: workflow_class, workflow_id, step_name,
68
+ # error_class (nil for a manual pause)
69
+ # pgbus.workflow_cancelled — a workflow run was cancelled
70
+ # payload: workflow_class, workflow_id, step_name
71
+ # pgbus.workflow.stall_detected — a step was stalled on the redelivery of the message that
72
+ # claimed it (detection: "redelivery": its worker died, or
73
+ # its heartbeat lapsed) or after it outlived its stall_at
74
+ # (detection: "sweep")
75
+ # payload: workflow_class, workflow_id, step_name, attempt,
76
+ # detection
77
+ # pgbus.workflow.signal — signal! stored a signal on a run (or found it stored:
78
+ # a redelivery with the same idempotency key)
79
+ # payload: workflow_class, workflow_id, name, duplicate,
80
+ # woke (true when it woke the step waiting for it)
81
+ # pgbus.workflow.wait_timed_out — a waiting step's wait_timeout: passed; its on_timeout
82
+ # applied (the sweep, or TestHelpers#expire_wait)
83
+ # payload: workflow_class, workflow_id, step_name, attempt,
84
+ # action (pause! | cancel! | skip! | run)
85
+ # pgbus.workflow.gauge — emitted by each dispatcher workflow sweep
86
+ # payload: statuses ([{ workflow_class:, status:, count: }]),
87
+ # stuck ([{ workflow_class:, reason:, count: }]), zero-filled
88
+ # across every known class; reasons include waiting_overdue
57
89
  #
58
90
  module Instrumentation
59
91
  module_function