pgbus 0.19.0 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +329 -147
- data/README.md +65 -27
- data/app/controllers/pgbus/deploy_tasks_controller.rb +38 -0
- data/app/helpers/pgbus/deploy_tasks_helper.rb +77 -0
- data/app/helpers/pgbus/processes_helper.rb +1 -1
- data/app/models/pgbus/deploy_task.rb +104 -0
- data/app/models/pgbus/deploy_task_run.rb +45 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/deploy_tasks/_rollover_banner.html.erb +25 -0
- data/app/views/pgbus/deploy_tasks/_tasks_table.html.erb +71 -0
- data/app/views/pgbus/deploy_tasks/index.html.erb +6 -0
- data/app/views/pgbus/deploy_tasks/show.html.erb +134 -0
- data/config/i18n-tasks.yml +2 -0
- data/config/locales/da.yml +97 -0
- data/config/locales/de.yml +97 -0
- data/config/locales/en.yml +98 -1
- data/config/locales/es.yml +97 -0
- data/config/locales/fi.yml +97 -0
- data/config/locales/fr.yml +97 -0
- data/config/locales/it.yml +97 -0
- data/config/locales/ja.yml +97 -0
- data/config/locales/nb.yml +97 -0
- data/config/locales/nl.yml +97 -0
- data/config/locales/pt.yml +97 -0
- data/config/locales/sv.yml +97 -0
- data/config/routes.rb +7 -0
- data/lib/generators/pgbus/add_batch_callback_jobs_generator.rb +4 -1
- data/lib/generators/pgbus/add_deploy_tasks_generator.rb +51 -0
- data/lib/generators/pgbus/templates/add_deploy_tasks_tables.rb.erb +43 -0
- data/lib/generators/pgbus/templates/deploy_tasks.yml.erb +42 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +43 -0
- data/lib/pgbus/active_job/batch_id.rb +10 -4
- data/lib/pgbus/batch/builder.rb +92 -0
- data/lib/pgbus/batch.rb +97 -22
- data/lib/pgbus/configuration/deploy_settings.rb +46 -0
- data/lib/pgbus/configuration.rb +3 -0
- data/lib/pgbus/deploy/revision.rb +59 -0
- data/lib/pgbus/deploy/rollover.rb +65 -0
- data/lib/pgbus/deploy/runner.rb +259 -0
- data/lib/pgbus/deploy/task.rb +105 -0
- data/lib/pgbus/engine.rb +17 -0
- data/lib/pgbus/generators/migration_detector.rb +12 -1
- data/lib/pgbus/instrumentation.rb +14 -0
- data/lib/pgbus/integrations/appsignal/subscriber.rb +15 -0
- data/lib/pgbus/mcp/server.rb +2 -1
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +86 -0
- data/lib/pgbus/mcp.rb +1 -0
- data/lib/pgbus/metrics/subscriber.rb +12 -0
- data/lib/pgbus/outbox/poller.rb +1 -0
- data/lib/pgbus/process/consumer.rb +57 -3
- data/lib/pgbus/process/dispatcher.rb +18 -1
- data/lib/pgbus/process/heartbeat.rb +7 -2
- data/lib/pgbus/process/supervisor.rb +35 -3
- data/lib/pgbus/process/worker.rb +1 -0
- data/lib/pgbus/recurring/scheduler.rb +54 -20
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/deploy_task_list.rb +114 -0
- data/lib/pgbus/web/data_source/list_counts.rb +2 -1
- data/lib/pgbus/web/data_source.rb +1 -0
- data/lib/pgbus/web/deploy_task_state.rb +51 -0
- data/lib/pgbus.rb +11 -0
- data/sig/pgbus/configuration.rbs +10 -0
- data/sig/pgbus.rbs +9 -0
- metadata +21 -1
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Deploy
|
|
5
|
+
# One declared deploy task (issue #536): an entry of
|
|
6
|
+
# config/deploy_tasks.yml. The sibling of Recurring::Task without a
|
|
7
|
+
# schedule: it runs once per (key, version) after a rollover, or after
|
|
8
|
+
# every rollover when every_deploy is set.
|
|
9
|
+
class Task
|
|
10
|
+
DEFAULT_VERSION = "1"
|
|
11
|
+
|
|
12
|
+
attr_reader :key, :class_name, :command, :version, :every_deploy, :queue_name,
|
|
13
|
+
:arguments, :priority, :description
|
|
14
|
+
|
|
15
|
+
def self.from_configuration(key, **options)
|
|
16
|
+
options = options.transform_keys(&:to_sym)
|
|
17
|
+
new(
|
|
18
|
+
key: key.to_s,
|
|
19
|
+
class_name: options[:class],
|
|
20
|
+
command: options[:command],
|
|
21
|
+
version: options.fetch(:version, DEFAULT_VERSION),
|
|
22
|
+
explicit_version: options.key?(:version),
|
|
23
|
+
every_deploy: options.fetch(:every_deploy, false),
|
|
24
|
+
queue_name: options[:queue],
|
|
25
|
+
arguments: options[:args].is_a?(Hash) ? [options[:args]] : Array(options[:args]),
|
|
26
|
+
priority: options.fetch(:priority, 0).to_i,
|
|
27
|
+
description: options[:description]
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def initialize(key:, class_name: nil, command: nil, version: DEFAULT_VERSION, explicit_version: false,
|
|
32
|
+
every_deploy: false, queue_name: nil, arguments: [], priority: 0, description: nil)
|
|
33
|
+
@key = key
|
|
34
|
+
@class_name = class_name
|
|
35
|
+
@command = command
|
|
36
|
+
@version = version.to_s.strip
|
|
37
|
+
@explicit_version = explicit_version
|
|
38
|
+
@every_deploy = every_deploy
|
|
39
|
+
@queue_name = queue_name
|
|
40
|
+
@arguments = arguments || []
|
|
41
|
+
@priority = priority || 0
|
|
42
|
+
@description = description
|
|
43
|
+
@errors = []
|
|
44
|
+
@warnings = []
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def valid?
|
|
48
|
+
@errors = []
|
|
49
|
+
@warnings = []
|
|
50
|
+
validate!
|
|
51
|
+
@errors.empty?
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def errors
|
|
55
|
+
valid? unless @validated
|
|
56
|
+
@errors
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def warnings
|
|
60
|
+
valid? unless @validated
|
|
61
|
+
@warnings
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# What an arm for this task is stamped with: its version, or the
|
|
65
|
+
# revision it runs on for an every_deploy task.
|
|
66
|
+
def armed_for(revision)
|
|
67
|
+
every_deploy ? revision : version
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def job_class
|
|
71
|
+
class_name&.safe_constantize
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def to_h
|
|
75
|
+
{
|
|
76
|
+
key: key, class_name: class_name, command: command, version: version, every_deploy: every_deploy,
|
|
77
|
+
queue_name: queue_name, arguments: arguments, priority: priority, description: description
|
|
78
|
+
}
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def validate!
|
|
84
|
+
@validated = true
|
|
85
|
+
@errors << "Key must not be blank" if key.to_s.strip.empty?
|
|
86
|
+
validate_target!
|
|
87
|
+
@errors << "Version must not be blank" if version.empty?
|
|
88
|
+
@errors << "every_deploy must be true or false" unless [true, false].include?(every_deploy)
|
|
89
|
+
return unless every_deploy == true && @explicit_version
|
|
90
|
+
|
|
91
|
+
@warnings << "every_deploy is set, so version is ignored (it runs after every rollover)"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def validate_target!
|
|
95
|
+
{ class: class_name, command: command }.each do |name, value|
|
|
96
|
+
@errors << "#{name} must be a String" unless value.nil? || value.is_a?(String)
|
|
97
|
+
end
|
|
98
|
+
has_class = !class_name.to_s.strip.empty?
|
|
99
|
+
has_command = !command.to_s.strip.empty?
|
|
100
|
+
@errors << "Either class or command is required" unless has_class || has_command
|
|
101
|
+
@errors << "Declare either class or command, not both" if has_class && has_command
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
data/lib/pgbus/engine.rb
CHANGED
|
@@ -58,6 +58,23 @@ module Pgbus
|
|
|
58
58
|
end
|
|
59
59
|
end
|
|
60
60
|
|
|
61
|
+
# config/deploy_tasks.yml (issue #536): same shape and loader as
|
|
62
|
+
# recurring.yml. Explicit deploy_tasks_files replace the default file, so
|
|
63
|
+
# this runs after the app's config/initializers set them.
|
|
64
|
+
initializer "pgbus.deploy_tasks", after: :load_config_initializers do |app|
|
|
65
|
+
config = Pgbus.configuration
|
|
66
|
+
next if config.deploy_tasks
|
|
67
|
+
|
|
68
|
+
files = config.deploy_tasks_files
|
|
69
|
+
default_path = app.root.join("config", "deploy_tasks.yml")
|
|
70
|
+
tasks = if files
|
|
71
|
+
Pgbus::Recurring::ConfigLoader.load_all(files)
|
|
72
|
+
elsif default_path.exist?
|
|
73
|
+
Pgbus::Recurring::ConfigLoader.load(default_path)
|
|
74
|
+
end
|
|
75
|
+
config.deploy_tasks = tasks if tasks.present?
|
|
76
|
+
end
|
|
77
|
+
|
|
61
78
|
initializer "pgbus.db" do
|
|
62
79
|
ActiveSupport.on_load(:active_record) do
|
|
63
80
|
Pgbus::BusRecord.connects_to(**Pgbus.configuration.connects_to) if Pgbus.configuration.connects_to
|
|
@@ -77,6 +77,7 @@ module Pgbus
|
|
|
77
77
|
add_queue_states: "pgbus:add_queue_states",
|
|
78
78
|
add_outbox: "pgbus:add_outbox",
|
|
79
79
|
add_recurring: "pgbus:add_recurring",
|
|
80
|
+
add_deploy_tasks: "pgbus:add_deploy_tasks",
|
|
80
81
|
add_failed_events_index: "pgbus:add_failed_events_index",
|
|
81
82
|
add_processed_event_completion: "pgbus:add_processed_event_completion",
|
|
82
83
|
add_batch_executions: "pgbus:add_batch_executions",
|
|
@@ -99,6 +100,7 @@ module Pgbus
|
|
|
99
100
|
add_queue_states: "queue states table (pause/resume)",
|
|
100
101
|
add_outbox: "outbox entries table (transactional outbox)",
|
|
101
102
|
add_recurring: "recurring tasks + executions tables",
|
|
103
|
+
add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
|
|
102
104
|
add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
|
|
103
105
|
add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
|
|
104
106
|
add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
|
|
@@ -127,6 +129,7 @@ module Pgbus
|
|
|
127
129
|
*queue_states_migrations,
|
|
128
130
|
*outbox_migrations,
|
|
129
131
|
*recurring_migrations,
|
|
132
|
+
*deploy_tasks_migrations,
|
|
130
133
|
*failed_events_index_migrations,
|
|
131
134
|
*processed_event_completion_migrations,
|
|
132
135
|
*batch_executions_migrations,
|
|
@@ -215,6 +218,14 @@ module Pgbus
|
|
|
215
218
|
[:add_recurring]
|
|
216
219
|
end
|
|
217
220
|
|
|
221
|
+
# Same shape as recurring_migrations: plain create_table in the
|
|
222
|
+
# template, so the generator is queued only while a table is absent.
|
|
223
|
+
def deploy_tasks_migrations
|
|
224
|
+
return [] if table_exists?("pgbus_deploy_tasks") && table_exists?("pgbus_deploy_task_runs")
|
|
225
|
+
|
|
226
|
+
[:add_deploy_tasks]
|
|
227
|
+
end
|
|
228
|
+
|
|
218
229
|
def failed_events_index_migrations
|
|
219
230
|
# pgbus_failed_events is created by pgbus:install, but the
|
|
220
231
|
# unique (queue_name, msg_id) index was added later via its own
|
|
@@ -252,7 +263,7 @@ module Pgbus
|
|
|
252
263
|
[:add_batch_executions]
|
|
253
264
|
end
|
|
254
265
|
|
|
255
|
-
# Configured callback instances (
|
|
266
|
+
# Configured callback instances (on_complete: MyJob.new.set(...)) need
|
|
256
267
|
# somewhere to store the serialized job — issue #415.
|
|
257
268
|
def batch_callback_jobs_migrations
|
|
258
269
|
return [] unless table_exists?("pgbus_batches")
|
|
@@ -21,6 +21,17 @@ module Pgbus
|
|
|
21
21
|
# payload: queue, job_class, msg_id, vt, extensions
|
|
22
22
|
# pgbus.event_processed — event handler succeeded
|
|
23
23
|
# pgbus.event_failed — event handler raised; carries :exception_object
|
|
24
|
+
# pgbus.event_dead_lettered — a consumer moved an event that exceeded max_retries
|
|
25
|
+
# to its DLQ; fires once, after the move
|
|
26
|
+
# payload: queue_name, routing_key, handler, event_id,
|
|
27
|
+
# read_ct, msg_id, error (the class of the last
|
|
28
|
+
# recorded failure: a handler's error, or the
|
|
29
|
+
# consumer's own parse error for a body it could
|
|
30
|
+
# not route; nil when none was recorded). Any of
|
|
31
|
+
# routing_key/handler/event_id can be nil: all
|
|
32
|
+
# three when the body is not a JSON object, a
|
|
33
|
+
# field the envelope lacks, and handler when no
|
|
34
|
+
# single subscriber in this process owns the queue
|
|
24
35
|
# pgbus.event_unrouted — a consumer read an event from a queue no
|
|
25
36
|
# subscriber in this process owns (stale topic
|
|
26
37
|
# binding); the message is archived
|
|
@@ -28,6 +39,9 @@ module Pgbus
|
|
|
28
39
|
# pgbus.stream.broadcast — stream broadcast (sync or deferred)
|
|
29
40
|
# pgbus.outbox.publish — outbox row created
|
|
30
41
|
# pgbus.recurring.enqueue — scheduler enqueued a due recurring task
|
|
42
|
+
# pgbus.deploy_task.enqueue — a deploy task enqueue was attempted (after a rollover or by
|
|
43
|
+
# Run now); payload: task, version, revision, triggered_by, plus
|
|
44
|
+
# :exception/:exception_object when the enqueue raised
|
|
31
45
|
# pgbus.worker.recycle — worker hit a recycle threshold (reason/jobs_processed/memory_mb/lifetime_seconds)
|
|
32
46
|
# pgbus.consumer.recycle — consumer hit a recycle threshold (same payload shape as pgbus.worker.recycle)
|
|
33
47
|
# pgbus.serializer.serialize — job/event serialization
|
|
@@ -60,6 +60,7 @@ module Pgbus
|
|
|
60
60
|
subscribe("pgbus.job_dead_lettered") { |event| on_job_dead_lettered(event) },
|
|
61
61
|
subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
|
|
62
62
|
subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
|
|
63
|
+
subscribe("pgbus.event_dead_lettered") { |event| on_event_dead_lettered(event) },
|
|
63
64
|
subscribe("pgbus.client.send_message") { |event| on_send_message(event) },
|
|
64
65
|
subscribe("pgbus.client.send_batch") { |event| on_send_batch(event) },
|
|
65
66
|
subscribe("pgbus.client.read_batch") { |event| on_read_batch(event) },
|
|
@@ -175,6 +176,20 @@ module Pgbus
|
|
|
175
176
|
::Appsignal.set_error(err) if err && ::Appsignal.respond_to?(:set_error)
|
|
176
177
|
end
|
|
177
178
|
|
|
179
|
+
# A counter only, like on_job_dead_lettered: nothing is raised at
|
|
180
|
+
# move time, and the handler's error already went out through
|
|
181
|
+
# pgbus.event_failed on the attempt that raised (issue #543). A body
|
|
182
|
+
# that is not JSON leaves handler and routing_key nil; compact drops
|
|
183
|
+
# those tags, as Metrics::Subscriber does.
|
|
184
|
+
def on_event_dead_lettered(event)
|
|
185
|
+
payload = event.payload
|
|
186
|
+
::Appsignal.increment_counter(
|
|
187
|
+
"#{METRIC_PREFIX}event_count",
|
|
188
|
+
1,
|
|
189
|
+
{ handler: payload[:handler], routing_key: payload[:routing_key], status: "dead_lettered" }.compact
|
|
190
|
+
)
|
|
191
|
+
end
|
|
192
|
+
|
|
178
193
|
# ── Client (PGMQ wrapper) ───────────────────────────────────────
|
|
179
194
|
|
|
180
195
|
def on_send_message(event)
|
data/lib/pgbus/mcp/server.rb
CHANGED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module MCP
|
|
5
|
+
module Tools
|
|
6
|
+
# Deploy tasks with their state and the rollover the scheduler waits on.
|
|
7
|
+
# Maps to DataSource#deploy_tasks and #deploy_rollover. Job arguments are
|
|
8
|
+
# not part of the list payload (and are dropped here if a data source
|
|
9
|
+
# ever adds them), so the tool never carries them.
|
|
10
|
+
class DeployTasksTool < BaseTool
|
|
11
|
+
tool_name "pgbus_deploy_tasks"
|
|
12
|
+
title "Pgbus Deploy Tasks"
|
|
13
|
+
description <<~DESC
|
|
14
|
+
Deploy tasks: one-time jobs the scheduler enqueues once every live pgbus process runs the new app revision. Lists each task's state (disabled, waiting, due, ran, never) and the rollover status the scheduler waits on.
|
|
15
|
+
Returns every task unless page or per_page is given; then it is
|
|
16
|
+
paginated (per_page defaults to 100, capped at 100). The response
|
|
17
|
+
carries page, per_page, total (capped at 10000 when paginated; the
|
|
18
|
+
exact row count when not) and has_more; per_page is null when
|
|
19
|
+
unpaginated.
|
|
20
|
+
DESC
|
|
21
|
+
|
|
22
|
+
MAX_PER_PAGE = 100
|
|
23
|
+
MAX_PAGE = 1_000
|
|
24
|
+
|
|
25
|
+
input_schema(
|
|
26
|
+
properties: {
|
|
27
|
+
page: { type: "integer", description: "1-based page number (default 1, max 1000).", minimum: 1 },
|
|
28
|
+
per_page: { type: "integer", description: "Rows per page (default 100 once paginating, max 100).", minimum: 1 }
|
|
29
|
+
},
|
|
30
|
+
required: []
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def self.call(page: nil, per_page: nil, server_context: nil)
|
|
34
|
+
data_source = data_source_from(server_context)
|
|
35
|
+
return unpaged(data_source) if page.nil? && per_page.nil?
|
|
36
|
+
|
|
37
|
+
per_page = (per_page || MAX_PER_PAGE).to_i.clamp(1, MAX_PER_PAGE)
|
|
38
|
+
page = (page || 1).to_i.clamp(1, MAX_PAGE)
|
|
39
|
+
tasks = data_source.deploy_tasks(page: page, per_page: per_page)
|
|
40
|
+
count = data_source.list_count(:deploy_tasks)
|
|
41
|
+
|
|
42
|
+
json_response(
|
|
43
|
+
payload(data_source, tasks).merge(
|
|
44
|
+
page: page, per_page: per_page, total: count.total,
|
|
45
|
+
has_more: more_pages?(count, page: page, per_page: per_page, shown: tasks.size)
|
|
46
|
+
)
|
|
47
|
+
)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def self.unpaged(data_source)
|
|
51
|
+
tasks = data_source.deploy_tasks
|
|
52
|
+
json_response(
|
|
53
|
+
payload(data_source, tasks).merge(page: 1, per_page: nil, total: tasks.size, has_more: false)
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
private_class_method :unpaged
|
|
57
|
+
|
|
58
|
+
def self.payload(data_source, tasks)
|
|
59
|
+
rollover = data_source.deploy_rollover
|
|
60
|
+
{ rollover: serialize(rollover), tasks: tasks.map { |row| task_hash(row, rollover) } }
|
|
61
|
+
end
|
|
62
|
+
private_class_method :payload
|
|
63
|
+
|
|
64
|
+
def self.task_hash(row, rollover)
|
|
65
|
+
state = Pgbus::Web::DeployTaskState.present(row, rollover)
|
|
66
|
+
serialize(row.except(:arguments)).merge(
|
|
67
|
+
"state" => state.state, "reason" => state.reason_key, "reason_args" => serialize(state.reason_args)
|
|
68
|
+
)
|
|
69
|
+
end
|
|
70
|
+
private_class_method :task_hash
|
|
71
|
+
|
|
72
|
+
# Times as ISO 8601 UTC, symbols as strings, string keys throughout.
|
|
73
|
+
def self.serialize(value)
|
|
74
|
+
case value
|
|
75
|
+
when Hash then value.to_h { |key, val| [key.to_s, serialize(val)] }
|
|
76
|
+
when Array then value.map { |val| serialize(val) }
|
|
77
|
+
when Symbol then value.to_s
|
|
78
|
+
when Time, DateTime, ActiveSupport::TimeWithZone then value.utc.iso8601
|
|
79
|
+
else value
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
private_class_method :serialize
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
data/lib/pgbus/mcp.rb
CHANGED
|
@@ -63,6 +63,7 @@ module Pgbus
|
|
|
63
63
|
subscribe("pgbus.job_visibility_extended") { |event| on_job_visibility_extended(event) },
|
|
64
64
|
subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
|
|
65
65
|
subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
|
|
66
|
+
subscribe("pgbus.event_dead_lettered") { |event| on_event_dead_lettered(event) },
|
|
66
67
|
subscribe("pgbus.event_skipped") { |event| on_event_skipped(event) },
|
|
67
68
|
subscribe("pgbus.client.send_message") { |event| on_send_message(event) },
|
|
68
69
|
subscribe("pgbus.client.send_batch") { |event| on_send_batch(event) },
|
|
@@ -154,6 +155,17 @@ module Pgbus
|
|
|
154
155
|
)
|
|
155
156
|
end
|
|
156
157
|
|
|
158
|
+
# The consumer moved an event that exhausted max_retries to its DLQ
|
|
159
|
+
# (issue #543). A body that is not JSON leaves handler and routing_key
|
|
160
|
+
# nil; compact drops those tags.
|
|
161
|
+
def on_event_dead_lettered(event)
|
|
162
|
+
payload = event.payload
|
|
163
|
+
backend.increment(
|
|
164
|
+
"#{METRIC_PREFIX}event_count", 1,
|
|
165
|
+
compact(handler: payload[:handler], routing_key: payload[:routing_key], status: "dead_lettered")
|
|
166
|
+
)
|
|
167
|
+
end
|
|
168
|
+
|
|
157
169
|
# An idempotent handler that did not run this delivery. `reason` is what
|
|
158
170
|
# makes the skip readable: :completed/:cached is deduplication working,
|
|
159
171
|
# :owned means a second delivery arrived while the holder was still
|
data/lib/pgbus/outbox/poller.rb
CHANGED
|
@@ -289,9 +289,9 @@ module Pgbus
|
|
|
289
289
|
@jobs_processed.increment
|
|
290
290
|
end
|
|
291
291
|
|
|
292
|
-
# Order: read the last error, move with the dead-letter block,
|
|
293
|
-
#
|
|
294
|
-
# next attempt still carries the reason.
|
|
292
|
+
# Order: read the last error, move with the dead-letter block, clear the
|
|
293
|
+
# failed-event row, then announce it. A move that raises keeps the row,
|
|
294
|
+
# so the next attempt still carries the reason, and announces nothing.
|
|
295
295
|
def dead_letter(message, queue_name)
|
|
296
296
|
last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
|
|
297
297
|
headers = DeadLetterHeader.build(
|
|
@@ -301,6 +301,59 @@ module Pgbus
|
|
|
301
301
|
)
|
|
302
302
|
Pgbus.client.move_to_dead_letter(queue_name, message, headers: headers)
|
|
303
303
|
FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
|
|
304
|
+
instrument_dead_lettered(message, queue_name, last_error)
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# pgbus.event_dead_lettered, the event twin of the executor's
|
|
308
|
+
# pgbus.job_dead_lettered (issue #543). The dead-letter branch runs
|
|
309
|
+
# before handle_message parses the envelope, so it is parsed here: once
|
|
310
|
+
# per dead-lettered message, never on the success path. The move has
|
|
311
|
+
# already happened, so nothing here may turn the dead-letter into a
|
|
312
|
+
# failure (a failed stat, a circuit-breaker trip): the guard is the
|
|
313
|
+
# executor's instrument rescue, widened on purpose to cover building the
|
|
314
|
+
# payload as well as the subscribers, and logs at the same debug level.
|
|
315
|
+
def instrument_dead_lettered(message, queue_name, last_error)
|
|
316
|
+
envelope = dead_letter_envelope(message)
|
|
317
|
+
routing_key = envelope_routing_key(envelope)
|
|
318
|
+
Pgbus::Instrumentation.instrument(
|
|
319
|
+
"pgbus.event_dead_lettered",
|
|
320
|
+
queue_name: queue_name, routing_key: routing_key,
|
|
321
|
+
handler: owning_handler_name(routing_key, queue_name), event_id: envelope["event_id"],
|
|
322
|
+
read_ct: message.read_ct.to_i, msg_id: message.msg_id.to_i, error: last_error&.dig(:error_class)
|
|
323
|
+
)
|
|
324
|
+
rescue StandardError => e
|
|
325
|
+
Pgbus.logger.debug { "[Pgbus] Notification failure pgbus.event_dead_lettered: #{e.class}: #{e.message}" }
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
# The body that killed a message may not be a JSON object at all: PGMQ
|
|
329
|
+
# stores jsonb, so a producer writing straight to the queue can leave a
|
|
330
|
+
# scalar or an array, and the success path's dig raises on those every
|
|
331
|
+
# read. Any of them reads as an empty envelope, so the notification
|
|
332
|
+
# still fires.
|
|
333
|
+
def dead_letter_envelope(message)
|
|
334
|
+
raw = JSON.parse(message.message)
|
|
335
|
+
raw.is_a?(Hash) ? raw : {}
|
|
336
|
+
rescue JSON::ParserError, TypeError
|
|
337
|
+
{}
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# The success path's lookup (headers first, then the top level),
|
|
341
|
+
# tolerating a headers value that is not an object.
|
|
342
|
+
def envelope_routing_key(envelope)
|
|
343
|
+
headers = envelope["headers"]
|
|
344
|
+
(headers["routing_key"] if headers.is_a?(Hash)) || envelope["routing_key"]
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# The handler of the one subscriber that owns +queue_name+ in this
|
|
348
|
+
# process, or nil when there is none, no usable routing key, or more than
|
|
349
|
+
# one: #469 makes one owner per queue the convention, but Registry does
|
|
350
|
+
# not enforce it, and dispatch runs every owner, so naming one of several
|
|
351
|
+
# would misattribute the dead letter.
|
|
352
|
+
def owning_handler_name(routing_key, queue_name)
|
|
353
|
+
return unless routing_key.is_a?(String)
|
|
354
|
+
|
|
355
|
+
owners = @registry.handlers_for(routing_key, queue_name: queue_name)
|
|
356
|
+
owners.first.handler_class.name if owners.one?
|
|
304
357
|
end
|
|
305
358
|
|
|
306
359
|
# The executor's handle_failure, for events (issue #494): one upsert per
|
|
@@ -615,6 +668,7 @@ module Pgbus
|
|
|
615
668
|
def start_heartbeat
|
|
616
669
|
@heartbeat = Heartbeat.new(
|
|
617
670
|
kind: "consumer",
|
|
671
|
+
config: @config,
|
|
618
672
|
metadata: {
|
|
619
673
|
topics: topics, threads: threads, pid: ::Process.pid, execution_mode: @execution_mode,
|
|
620
674
|
started_at: @started_at.to_f, supervisor_pid: ::Process.ppid
|
|
@@ -13,6 +13,7 @@ module Pgbus
|
|
|
13
13
|
CONCURRENCY_INTERVAL = 300 # Run concurrency cleanup every 5 minutes
|
|
14
14
|
BATCH_CLEANUP_INTERVAL = 3600 # Run batch cleanup every hour
|
|
15
15
|
RECURRING_CLEANUP_INTERVAL = 3600 # Run recurring execution cleanup every hour
|
|
16
|
+
DEPLOY_RUN_CLEANUP_INTERVAL = 3600 # Run deploy task run cleanup every hour
|
|
16
17
|
ARCHIVE_COMPACTION_INTERVAL = 3600 # Run archive compaction every hour
|
|
17
18
|
OUTBOX_CLEANUP_INTERVAL = 3600 # Run outbox cleanup every hour
|
|
18
19
|
JOB_LOCK_CLEANUP_INTERVAL = 300 # Run job lock cleanup every 5 minutes
|
|
@@ -52,7 +53,8 @@ module Pgbus
|
|
|
52
53
|
MAINTENANCE_TIMESTAMPS = %i[
|
|
53
54
|
@last_cleanup_at @last_reap_at @last_concurrency_at @last_batch_cleanup_at
|
|
54
55
|
@last_batch_sweep_at
|
|
55
|
-
@last_recurring_cleanup_at @
|
|
56
|
+
@last_recurring_cleanup_at @last_deploy_run_cleanup_at @last_archive_compaction_at
|
|
57
|
+
@last_stream_archive_compaction_at
|
|
56
58
|
@last_outbox_cleanup_at @last_job_lock_cleanup_at @last_stats_cleanup_at
|
|
57
59
|
@last_orphan_stream_sweep_at @last_table_maintenance_at
|
|
58
60
|
].freeze
|
|
@@ -236,6 +238,7 @@ module Pgbus
|
|
|
236
238
|
results << run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
|
|
237
239
|
results << run_if_due(now, :@last_batch_sweep_at, config.batch_sweep_interval) { sweep_stalled_batches }
|
|
238
240
|
results << run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
|
|
241
|
+
results << run_if_due(now, :@last_deploy_run_cleanup_at, DEPLOY_RUN_CLEANUP_INTERVAL) { cleanup_deploy_task_runs }
|
|
239
242
|
results << run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
|
|
240
243
|
results << run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
|
|
241
244
|
results << run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
|
|
@@ -681,6 +684,19 @@ module Pgbus
|
|
|
681
684
|
e
|
|
682
685
|
end
|
|
683
686
|
|
|
687
|
+
# Guarded on the table: deploy tasks default on, and an install that has
|
|
688
|
+
# not run pgbus:add_deploy_tasks must not warn every hour.
|
|
689
|
+
def cleanup_deploy_task_runs
|
|
690
|
+
retention = config.deploy_task_run_retention
|
|
691
|
+
return unless retention&.positive? && DeployTaskRun.table_exists?
|
|
692
|
+
|
|
693
|
+
deleted = DeployTaskRun.prunable_before(Time.current - retention).delete_all
|
|
694
|
+
Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old deploy task runs" } if deleted.positive?
|
|
695
|
+
rescue StandardError => e
|
|
696
|
+
log_maintenance_failure("Deploy task run cleanup", e)
|
|
697
|
+
e
|
|
698
|
+
end
|
|
699
|
+
|
|
684
700
|
def monotonic_now
|
|
685
701
|
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
686
702
|
end
|
|
@@ -697,6 +713,7 @@ module Pgbus
|
|
|
697
713
|
def start_heartbeat
|
|
698
714
|
@heartbeat = Heartbeat.new(
|
|
699
715
|
kind: "dispatcher",
|
|
716
|
+
config: @config,
|
|
700
717
|
metadata: { pid: ::Process.pid, started_at: Time.now.to_f, supervisor_pid: ::Process.ppid },
|
|
701
718
|
loop_tick_supplier: -> { @loop_tick_at.get }
|
|
702
719
|
)
|
|
@@ -11,9 +11,14 @@ module Pgbus
|
|
|
11
11
|
|
|
12
12
|
attr_reader :process_entry
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
# config: the owning process's configuration, for the app revision.
|
|
15
|
+
def initialize(kind:, metadata: {}, on_beat: nil, loop_tick_supplier: nil, metadata_supplier: nil,
|
|
16
|
+
config: Pgbus.configuration)
|
|
15
17
|
@kind = kind
|
|
16
|
-
|
|
18
|
+
# Every kind's row carries the app revision from its first
|
|
19
|
+
# registration on, nil included: the deploy-task rollover gate reads
|
|
20
|
+
# it (issue #536), and beat_metadata dups this hash so it survives.
|
|
21
|
+
@metadata = metadata.merge(revision: Deploy::Revision.current(config))
|
|
17
22
|
@on_beat = on_beat
|
|
18
23
|
@loop_tick_supplier = loop_tick_supplier
|
|
19
24
|
@metadata_supplier = metadata_supplier
|
|
@@ -425,9 +425,12 @@ module Pgbus
|
|
|
425
425
|
ErrorReporter.report(e, { action: "fork_dispatcher" })
|
|
426
426
|
end
|
|
427
427
|
|
|
428
|
+
# The scheduler runs recurring tasks and deploy tasks (issue #536);
|
|
429
|
+
# either one being enabled and configured forks it.
|
|
428
430
|
def boot_scheduler
|
|
429
|
-
|
|
430
|
-
|
|
431
|
+
recurring = config.recurring_enabled && recurring_tasks_configured?
|
|
432
|
+
deploy = config.deploy_tasks_enabled && deploy_tasks_configured?
|
|
433
|
+
return unless recurring || deploy
|
|
431
434
|
|
|
432
435
|
fork_scheduler
|
|
433
436
|
end
|
|
@@ -438,7 +441,8 @@ module Pgbus
|
|
|
438
441
|
restore_signals
|
|
439
442
|
setup_child_process
|
|
440
443
|
load_rails_app
|
|
441
|
-
load_recurring_config
|
|
444
|
+
load_recurring_config if config.recurring_enabled
|
|
445
|
+
load_deploy_tasks_config if config.deploy_tasks_enabled
|
|
442
446
|
bootstrap_queues!
|
|
443
447
|
scheduler = Recurring::Scheduler.new(config: config)
|
|
444
448
|
scheduler.run
|
|
@@ -488,6 +492,33 @@ module Pgbus
|
|
|
488
492
|
config.recurring_tasks = Recurring::ConfigLoader.load(path)
|
|
489
493
|
end
|
|
490
494
|
|
|
495
|
+
def deploy_tasks_configured?
|
|
496
|
+
return true if config.deploy_tasks&.any?
|
|
497
|
+
# Explicit files replace the default file, as in load_deploy_tasks_config.
|
|
498
|
+
return config.deploy_tasks_files.any? { |f| File.exist?(f.to_s) } if config.deploy_tasks_files
|
|
499
|
+
|
|
500
|
+
path = default_deploy_tasks_path
|
|
501
|
+
path ? File.exist?(path.to_s) : false
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# The engine initializer loads config/deploy_tasks.yml in a Rails boot;
|
|
505
|
+
# this covers a scheduler fork whose config was not loaded that way.
|
|
506
|
+
def load_deploy_tasks_config
|
|
507
|
+
return if config.deploy_tasks&.any?
|
|
508
|
+
|
|
509
|
+
files = config.deploy_tasks_files
|
|
510
|
+
tasks = if files
|
|
511
|
+
Recurring::ConfigLoader.load_all(files)
|
|
512
|
+
elsif (path = default_deploy_tasks_path) && File.exist?(path.to_s)
|
|
513
|
+
Recurring::ConfigLoader.load(path)
|
|
514
|
+
end
|
|
515
|
+
config.deploy_tasks = tasks if tasks&.any?
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
def default_deploy_tasks_path
|
|
519
|
+
Rails.root.join("config", "deploy_tasks.yml") if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
|
|
520
|
+
end
|
|
521
|
+
|
|
491
522
|
def boot_consumers
|
|
492
523
|
return unless config.event_consumers
|
|
493
524
|
|
|
@@ -903,6 +934,7 @@ module Pgbus
|
|
|
903
934
|
def start_heartbeat
|
|
904
935
|
@heartbeat = Heartbeat.new(
|
|
905
936
|
kind: "supervisor",
|
|
937
|
+
config: @config,
|
|
906
938
|
metadata: { pid: ::Process.pid, hostname: Socket.gethostname, started_at: Time.now.to_f }
|
|
907
939
|
)
|
|
908
940
|
@heartbeat.start
|
data/lib/pgbus/process/worker.rb
CHANGED