pgbus 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +59 -6
- data/README.md +3 -2
- data/app/frontend/pgbus/modules/charts.js +18 -0
- data/app/frontend/pgbus/modules/charts.min.js +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/helpers/pgbus/workflow_insights_helper.rb +8 -0
- data/app/helpers/pgbus/workflows_helper.rb +2 -2
- data/app/models/pgbus/blocked_execution.rb +12 -0
- data/app/models/pgbus/uniqueness_key.rb +88 -3
- data/app/models/pgbus/workflow_entry.rb +3 -2
- data/app/models/pgbus/workflow_step.rb +7 -0
- data/app/views/pgbus/workflows/_timeline.html.erb +11 -1
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +3 -0
- data/config/locales/da.yml +7 -1
- data/config/locales/de.yml +7 -1
- data/config/locales/en.yml +7 -1
- data/config/locales/es.yml +7 -1
- data/config/locales/fi.yml +7 -1
- data/config/locales/fr.yml +7 -1
- data/config/locales/it.yml +7 -1
- data/config/locales/ja.yml +7 -1
- data/config/locales/nb.yml +7 -1
- data/config/locales/nl.yml +7 -1
- data/config/locales/pt.yml +7 -1
- data/config/locales/sv.yml +7 -1
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +3 -1
- data/lib/generators/pgbus/templates/migration.rb.erb +3 -1
- data/lib/pgbus/active_job/adapter.rb +81 -3
- data/lib/pgbus/active_job/delivery.rb +20 -6
- data/lib/pgbus/active_job/executor.rb +18 -8
- data/lib/pgbus/concurrency/blocked_execution.rb +14 -6
- data/lib/pgbus/configuration/workflow_settings.rb +1 -1
- data/lib/pgbus/configuration.rb +43 -1
- data/lib/pgbus/dead_letter_header.rb +9 -1
- data/lib/pgbus/instrumentation.rb +3 -2
- data/lib/pgbus/integrations/appsignal/dashboard.json +30 -1
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +1 -1
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +1 -1
- data/lib/pgbus/metrics/workflow_metrics.rb +11 -1
- data/lib/pgbus/process/consumer.rb +2 -1
- data/lib/pgbus/process/dispatcher.rb +13 -3
- data/lib/pgbus/process/worker.rb +10 -1
- data/lib/pgbus/recurring/schedule.rb +1 -1
- data/lib/pgbus/streams/presence.rb +65 -36
- data/lib/pgbus/uniqueness.rb +23 -6
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_list.rb +52 -7
- data/lib/pgbus/web/data_source.rb +123 -15
- data/lib/pgbus/web/workflow_insights_report.rb +11 -3
- data/lib/pgbus/web/workflow_state.rb +19 -3
- data/lib/pgbus/web/workflow_timeline.rb +23 -6
- data/lib/pgbus/workflow/instance.rb +36 -3
- data/lib/pgbus/workflow/runner.rb +2 -2
- data/lib/pgbus/workflow/signal_peek.rb +47 -0
- data/lib/pgbus/workflow/signals.rb +31 -10
- data/lib/pgbus/workflow/step_definition.rb +49 -5
- data/lib/pgbus/workflow/step_execution.rb +27 -10
- data/lib/pgbus/workflow/wait_timeout.rb +56 -4
- data/lib/pgbus/workflow.rb +11 -5
- data/sig/pgbus/configuration.rbs +1 -0
- data/sig/pgbus.rbs +43 -0
- metadata +2 -1
|
@@ -33,7 +33,7 @@ module Pgbus
|
|
|
33
33
|
FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
|
|
34
34
|
signal_concurrency(payload)
|
|
35
35
|
signal_batch_discarded(payload)
|
|
36
|
-
|
|
36
|
+
release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message)
|
|
37
37
|
record_stat(payload, queue_name, "dead_lettered", execution_start, message: message)
|
|
38
38
|
instrument(
|
|
39
39
|
"pgbus.job_dead_lettered",
|
|
@@ -100,7 +100,10 @@ module Pgbus
|
|
|
100
100
|
assign_batch_id(job, payload)
|
|
101
101
|
Pgbus.logger.debug { "[Pgbus::Executor] running #{log_tag(message, queue_name)} job_class=#{job_class}" }
|
|
102
102
|
with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) do
|
|
103
|
-
Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name
|
|
103
|
+
Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name,
|
|
104
|
+
enqueued_at: message.enqueued_at, job_id: payload["job_id"]) do
|
|
105
|
+
execute_job(job)
|
|
106
|
+
end
|
|
104
107
|
end
|
|
105
108
|
# retry_on re-enqueues from inside perform_now and returns normally:
|
|
106
109
|
# this attempt is done (archive it) but the job is not — the retry
|
|
@@ -121,7 +124,7 @@ module Pgbus
|
|
|
121
124
|
end
|
|
122
125
|
Pgbus.logger.debug { "[Pgbus::Executor] archived #{log_tag(message, queue_name)} job_class=#{job_class}" }
|
|
123
126
|
job_succeeded = true
|
|
124
|
-
release_uniqueness_lock(uniqueness_key)
|
|
127
|
+
release_uniqueness_lock(uniqueness_key, queue_name, message)
|
|
125
128
|
# Only FailedEventRecorder.record! writes this table, and only after a
|
|
126
129
|
# failed attempt, so a first delivery has no row: skip the per-job
|
|
127
130
|
# DELETE round trip (issue #484). A stale row from a dropped-and-
|
|
@@ -146,7 +149,7 @@ module Pgbus
|
|
|
146
149
|
# A failed :while_executing attempt is no longer executing — release
|
|
147
150
|
# so the retry (same message, after VT) can acquire. :until_executed
|
|
148
151
|
# keeps its lock until success/DLQ by design (#126, #333).
|
|
149
|
-
release_uniqueness_lock(uniqueness_key) if uniqueness_strategy == :while_executing
|
|
152
|
+
release_uniqueness_lock(uniqueness_key, queue_name, message) if uniqueness_strategy == :while_executing
|
|
150
153
|
instrument(
|
|
151
154
|
"pgbus.job_failed",
|
|
152
155
|
queue: queue_name,
|
|
@@ -192,10 +195,15 @@ module Pgbus
|
|
|
192
195
|
job.batch_id = batch_id
|
|
193
196
|
end
|
|
194
197
|
|
|
195
|
-
|
|
198
|
+
# Releases only the row this message holds, never the key as such: a
|
|
199
|
+
# row the job moved onto its retry (issue #564), or a successor's, stays.
|
|
200
|
+
# The queue is spelled as the row was bound (Uniqueness.lock_queue_name),
|
|
201
|
+
# as for release_duplicate_execution_lock.
|
|
202
|
+
def release_uniqueness_lock(key, queue_name, message)
|
|
196
203
|
return unless key
|
|
197
204
|
|
|
198
|
-
Uniqueness.
|
|
205
|
+
UniquenessKey.release_held_by!(key, queue_name: Uniqueness.lock_queue_name(queue_name, config: config),
|
|
206
|
+
msg_id: message.msg_id.to_i, enqueued_at: message.enqueued_at)
|
|
199
207
|
rescue StandardError => e
|
|
200
208
|
Pgbus.logger.warn { "[Pgbus] Uniqueness release failed: #{e.message}" }
|
|
201
209
|
end
|
|
@@ -425,7 +433,8 @@ module Pgbus
|
|
|
425
433
|
|
|
426
434
|
# Same queue the execution lock was acquired under, so the match
|
|
427
435
|
# cannot land on another queue's message of the same id.
|
|
428
|
-
UniquenessKey.release_if_bound!(uniqueness_key, queue_name: queue_name,
|
|
436
|
+
UniquenessKey.release_if_bound!(uniqueness_key, queue_name: Uniqueness.lock_queue_name(queue_name, config: config),
|
|
437
|
+
msg_id: msg_id)
|
|
429
438
|
rescue StandardError => e
|
|
430
439
|
Pgbus.logger.warn { "[Pgbus] Duplicate-execution lock release failed: #{e.message}" }
|
|
431
440
|
end
|
|
@@ -442,7 +451,8 @@ module Pgbus
|
|
|
442
451
|
end
|
|
443
452
|
headers = DeadLetterHeader.build(
|
|
444
453
|
existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "worker",
|
|
445
|
-
source_queue: source_queue || config.queue_name(queue_name),
|
|
454
|
+
source_queue: source_queue || config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
|
|
455
|
+
source_enqueued_at: message.enqueued_at, attempts: message.read_ct.to_i,
|
|
446
456
|
max_retries: config.max_retries, error: last_error
|
|
447
457
|
)
|
|
448
458
|
if source_queue
|
|
@@ -16,13 +16,21 @@ module Pgbus
|
|
|
16
16
|
# once. Handing it a pre-serialized String stored a JSON *string*
|
|
17
17
|
# (double-encoded), which every document reader of the column —
|
|
18
18
|
# `payload->>'job_id'`, the job-class lookup, `scheduled_at` — misread.
|
|
19
|
+
#
|
|
20
|
+
# created_at is left to the column's CURRENT_TIMESTAMP default: the
|
|
21
|
+
# job's uniqueness lock is stamped by the database too, and discarding
|
|
22
|
+
# a parked job releases only a lock acquired no later than the park
|
|
23
|
+
# (UniquenessKey.release_if_unbound!), so both sides need one clock.
|
|
19
24
|
def insert(concurrency_key:, queue_name:, payload:, duration:, priority: 0)
|
|
20
|
-
Pgbus::BlockedExecution.
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
Pgbus::BlockedExecution.insert!(
|
|
26
|
+
{
|
|
27
|
+
concurrency_key: concurrency_key,
|
|
28
|
+
queue_name: queue_name,
|
|
29
|
+
payload: payload,
|
|
30
|
+
priority: priority,
|
|
31
|
+
expires_at: Time.current + duration
|
|
32
|
+
},
|
|
33
|
+
record_timestamps: false
|
|
26
34
|
)
|
|
27
35
|
end
|
|
28
36
|
|
|
@@ -12,7 +12,7 @@ module Pgbus
|
|
|
12
12
|
# config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
|
|
13
13
|
# config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
|
|
14
14
|
# config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
|
|
15
|
-
# #
|
|
15
|
+
# # deadline counts as stuck (waiting_overdue)
|
|
16
16
|
# config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
|
|
17
17
|
# # the dashboard's subject link (nil: plain text)
|
|
18
18
|
module WorkflowSettings
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -13,7 +13,8 @@ module Pgbus
|
|
|
13
13
|
attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
|
|
14
14
|
|
|
15
15
|
# Queue settings
|
|
16
|
-
attr_accessor :default_queue
|
|
16
|
+
attr_accessor :default_queue
|
|
17
|
+
attr_reader :queue_prefix # rubocop:disable Style/AccessorGrouping -- the writer normalizes (see queue_prefix=)
|
|
17
18
|
|
|
18
19
|
# Worker settings
|
|
19
20
|
attr_accessor :polling_interval, :prefetch_limit, :execution_mode
|
|
@@ -597,6 +598,25 @@ module Pgbus
|
|
|
597
598
|
@finalized = false
|
|
598
599
|
end
|
|
599
600
|
|
|
601
|
+
# Stored the way the queue tables spell it ("my-app" → "my_app"). Table
|
|
602
|
+
# names are unchanged (queue_name normalizes the joined name the same
|
|
603
|
+
# way), but code that strips the prefix off a table name to get its queue
|
|
604
|
+
# back (the dashboard, the worker, a job's uniqueness lock queue) then
|
|
605
|
+
# finds it. A value normalize cannot spell (nothing table-safe left, or
|
|
606
|
+
# too long) is kept as given, as before: validate! rejects one that is
|
|
607
|
+
# too long, and queue_name drops one with nothing table-safe in it.
|
|
608
|
+
def queue_prefix=(value)
|
|
609
|
+
@queue_prefix = if value.is_a?(String) && !value.empty?
|
|
610
|
+
begin
|
|
611
|
+
QueueNameValidator.normalize(value)
|
|
612
|
+
rescue ArgumentError
|
|
613
|
+
value
|
|
614
|
+
end
|
|
615
|
+
else
|
|
616
|
+
value
|
|
617
|
+
end
|
|
618
|
+
end
|
|
619
|
+
|
|
600
620
|
def queue_name(name)
|
|
601
621
|
full = "#{queue_prefix}_#{name}"
|
|
602
622
|
QueueNameValidator.normalize(full)
|
|
@@ -1002,6 +1022,27 @@ module Pgbus
|
|
|
1002
1022
|
raise Pgbus::ConfigurationError, "capsule processes must be a positive Integer (got #{value.inspect})"
|
|
1003
1023
|
end
|
|
1004
1024
|
|
|
1025
|
+
# A prefix with a leading, trailing or doubled underscore is kept as
|
|
1026
|
+
# written (its tables already exist under it), but queue_name collapses
|
|
1027
|
+
# the separator after it only for a queue name that needs normalizing:
|
|
1028
|
+
# "myapp_" gives myapp__orders yet myapp_orders_handler for
|
|
1029
|
+
# "orders-handler". A job's own release and the dashboard derive the same
|
|
1030
|
+
# lock queue for it and still match, but the reaper cannot resolve that
|
|
1031
|
+
# name back to the table, so a lock orphaned there is never reclaimed and
|
|
1032
|
+
# has to be released by key. Legal, but warned.
|
|
1033
|
+
def warn_queue_prefix_underscores
|
|
1034
|
+
return unless queue_prefix.match?(/\A_|_\z|__/)
|
|
1035
|
+
|
|
1036
|
+
suggested = queue_prefix.squeeze("_").delete_prefix("_").delete_suffix("_")
|
|
1037
|
+
suggestion = suggested.empty? ? "" : "; prefer #{suggested.inspect} for new deployments"
|
|
1038
|
+
Pgbus.logger.warn do
|
|
1039
|
+
"[Pgbus] queue_prefix #{queue_prefix.inspect} has a leading, trailing or doubled underscore — a queue " \
|
|
1040
|
+
"whose name needs normalizing (\"orders-handler\") gets a table spelled unlike its queue, so the " \
|
|
1041
|
+
"reaper cannot find that table: a uniqueness lock orphaned there (a worker crash before its release) " \
|
|
1042
|
+
"is never reclaimed and has to be released by key#{suggestion}"
|
|
1043
|
+
end
|
|
1044
|
+
end
|
|
1045
|
+
|
|
1005
1046
|
# Buffered claims are kept invisible only by the visibility heartbeat.
|
|
1006
1047
|
# Without it a message buffered longer than visibility_timeout can be
|
|
1007
1048
|
# claimed by another worker while it still waits here. Legal, but warned.
|
|
@@ -1075,6 +1116,7 @@ module Pgbus
|
|
|
1075
1116
|
{ queue_prefix: queue_prefix, default_queue: default_queue }.each do |name, value|
|
|
1076
1117
|
raise Pgbus::ConfigurationError, "#{name} must be a non-empty String" unless value.is_a?(String) && !value.empty?
|
|
1077
1118
|
end
|
|
1119
|
+
warn_queue_prefix_underscores
|
|
1078
1120
|
|
|
1079
1121
|
# web_auth gates the dashboard; a non-callable value silently leaves it
|
|
1080
1122
|
# open (mirror streams_presence_member).
|
|
@@ -30,7 +30,13 @@ module Pgbus
|
|
|
30
30
|
|
|
31
31
|
# error: FailedEventRecorder.last_error's Hash, or nil when no failure
|
|
32
32
|
# was recorded. Returns the merged headers as a JSON string.
|
|
33
|
-
|
|
33
|
+
# source_msg_id / source_enqueued_at: the message's id in source_queue and
|
|
34
|
+
# when it was sent (the copy gets a new id and enqueued_at in the DLQ).
|
|
35
|
+
# A discard of the copy uses both to release only the uniqueness lock the
|
|
36
|
+
# original still holds: bound to it, or unbound and acquired before its
|
|
37
|
+
# send, never a lock it handed to a retry after (issue #564).
|
|
38
|
+
def build(existing:, reason:, source:, source_queue:, attempts:, max_retries:, error: nil, source_msg_id: nil,
|
|
39
|
+
source_enqueued_at: nil, now: Time.now.utc)
|
|
34
40
|
headers = decode(existing, warn: true)
|
|
35
41
|
retries = count(headers.delete(RETRIES_KEY))
|
|
36
42
|
block = {
|
|
@@ -42,6 +48,8 @@ module Pgbus
|
|
|
42
48
|
"max_retries" => max_retries,
|
|
43
49
|
"dead_lettered_at" => iso(now)
|
|
44
50
|
}
|
|
51
|
+
block["source_msg_id"] = source_msg_id if source_msg_id
|
|
52
|
+
block["source_enqueued_at"] = iso(source_enqueued_at) if source_enqueued_at
|
|
45
53
|
block["retries_from_dlq"] = retries if retries.positive?
|
|
46
54
|
block.merge!(error_fields(error)) if error
|
|
47
55
|
JSON.generate(headers.merge(KEY => block))
|
|
@@ -79,9 +79,10 @@ module Pgbus
|
|
|
79
79
|
# payload: workflow_class, workflow_id, name, duplicate,
|
|
80
80
|
# woke (true when it woke the step waiting for it)
|
|
81
81
|
# pgbus.workflow.wait_timed_out — a waiting step's wait_timeout: passed; its on_timeout
|
|
82
|
-
# applied (the sweep, or TestHelpers#expire_wait)
|
|
82
|
+
# applied (the sweep, or TestHelpers#expire_wait); under
|
|
83
|
+
# notify the step keeps waiting, once per sweep that finds it late
|
|
83
84
|
# payload: workflow_class, workflow_id, step_name, attempt,
|
|
84
|
-
# action (pause! | cancel! | skip! | run)
|
|
85
|
+
# action (pause! | cancel! | skip! | run | notify)
|
|
85
86
|
# pgbus.workflow.gauge — emitted by each dispatcher workflow sweep
|
|
86
87
|
# payload: statuses ([{ workflow_class:, status:, count: }]),
|
|
87
88
|
# stuck ([{ workflow_class:, reason:, count: }]), zero-filled
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
],
|
|
5
5
|
"dashboard": {
|
|
6
6
|
"title": "Pgbus",
|
|
7
|
-
"description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\nWorkflow step outcomes and run times, and
|
|
7
|
+
"description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\nWorkflow step outcomes and run times, stuck workflows and late waits.\n",
|
|
8
8
|
"visuals": [
|
|
9
9
|
{
|
|
10
10
|
"type": "timeseries",
|
|
@@ -520,6 +520,35 @@
|
|
|
520
520
|
]
|
|
521
521
|
}
|
|
522
522
|
]
|
|
523
|
+
},
|
|
524
|
+
{
|
|
525
|
+
"type": "timeseries",
|
|
526
|
+
"display": "LINE",
|
|
527
|
+
"title": "Late workflow waits",
|
|
528
|
+
"description": "Deadlines an on_timeout: :notify wait missed, by class and step: the run keeps waiting for its signal.",
|
|
529
|
+
"line_label": "%workflow_class% - %step%",
|
|
530
|
+
"format": "number",
|
|
531
|
+
"draw_null_as_zero": true,
|
|
532
|
+
"metrics": [
|
|
533
|
+
{
|
|
534
|
+
"name": "pgbus_workflow_wait_overdue_total",
|
|
535
|
+
"fields": [
|
|
536
|
+
{
|
|
537
|
+
"field": "COUNTER"
|
|
538
|
+
}
|
|
539
|
+
],
|
|
540
|
+
"tags": [
|
|
541
|
+
{
|
|
542
|
+
"key": "workflow_class",
|
|
543
|
+
"value": "*"
|
|
544
|
+
},
|
|
545
|
+
{
|
|
546
|
+
"key": "step",
|
|
547
|
+
"value": "*"
|
|
548
|
+
}
|
|
549
|
+
]
|
|
550
|
+
}
|
|
551
|
+
]
|
|
523
552
|
}
|
|
524
553
|
]
|
|
525
554
|
}
|
|
@@ -11,7 +11,7 @@ module Pgbus
|
|
|
11
11
|
tool_name "pgbus_workflow_stats"
|
|
12
12
|
title "Pgbus Workflow Stats"
|
|
13
13
|
description <<~DESC
|
|
14
|
-
One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
|
|
14
|
+
One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): wait_timeout (a wait_for: step's wait_timeout: in seconds, "dynamic" when a lambda or a method name sets it per run, null without one), counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
|
|
15
15
|
DESC
|
|
16
16
|
|
|
17
17
|
input_schema(
|
|
@@ -9,7 +9,7 @@ module Pgbus
|
|
|
9
9
|
tool_name "pgbus_workflows_attention"
|
|
10
10
|
title "Pgbus Workflows Needing Attention"
|
|
11
11
|
description <<~DESC
|
|
12
|
-
Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a wait_timeout). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
|
|
12
|
+
Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a deadline: no wait_timeout, or a per-run one that set none or cleared it after a :notify notice). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
|
|
13
13
|
DESC
|
|
14
14
|
|
|
15
15
|
input_schema(
|
|
@@ -12,6 +12,7 @@ module Pgbus
|
|
|
12
12
|
# pgbus_workflows_paused_total{workflow_class, error_class} counter (error_class absent for a manual pause)
|
|
13
13
|
# pgbus_workflows_cancelled_total{workflow_class} counter
|
|
14
14
|
# pgbus_workflow_stalls_total{workflow_class, step, detection} counter (detection: redelivery | sweep)
|
|
15
|
+
# pgbus_workflow_wait_overdue_total{workflow_class, step} counter (a late notice of an on_timeout: :notify wait)
|
|
15
16
|
# pgbus_workflows{workflow_class, status} gauge, from each sweep
|
|
16
17
|
# pgbus_workflows_stuck{workflow_class, reason} gauge, from each sweep
|
|
17
18
|
#
|
|
@@ -20,7 +21,7 @@ module Pgbus
|
|
|
20
21
|
module WorkflowMetrics
|
|
21
22
|
EVENTS = %w[
|
|
22
23
|
pgbus.workflow.step pgbus.workflow_finished pgbus.workflow_paused pgbus.workflow_cancelled
|
|
23
|
-
pgbus.workflow.stall_detected pgbus.workflow.gauge
|
|
24
|
+
pgbus.workflow.stall_detected pgbus.workflow.wait_timed_out pgbus.workflow.gauge
|
|
24
25
|
].freeze
|
|
25
26
|
|
|
26
27
|
module_function
|
|
@@ -37,6 +38,7 @@ module Pgbus
|
|
|
37
38
|
when "pgbus.workflow_cancelled" then [counter("pgbus_workflows_cancelled_total", workflow_class: klass)]
|
|
38
39
|
when "pgbus.workflow.stall_detected"
|
|
39
40
|
[counter("pgbus_workflow_stalls_total", workflow_class: klass, step: payload[:step_name], detection: payload[:detection])]
|
|
41
|
+
when "pgbus.workflow.wait_timed_out" then wait_timed_out_samples(klass, payload)
|
|
40
42
|
when "pgbus.workflow.gauge" then gauge_samples(payload)
|
|
41
43
|
else []
|
|
42
44
|
end
|
|
@@ -49,6 +51,14 @@ module Pgbus
|
|
|
49
51
|
samples
|
|
50
52
|
end
|
|
51
53
|
|
|
54
|
+
# Only :notify keeps waiting past the deadline; every other on_timeout
|
|
55
|
+
# ends the wait and is counted by the step or run it ends.
|
|
56
|
+
def wait_timed_out_samples(klass, payload)
|
|
57
|
+
return [] unless payload[:action] == "notify"
|
|
58
|
+
|
|
59
|
+
[counter("pgbus_workflow_wait_overdue_total", workflow_class: klass, step: payload[:step_name])]
|
|
60
|
+
end
|
|
61
|
+
|
|
52
62
|
def gauge_samples(payload)
|
|
53
63
|
statuses = Array(payload[:statuses]).map do |row|
|
|
54
64
|
[:gauge, "pgbus_workflows", row[:count], { workflow_class: row[:workflow_class], status: row[:status] }]
|
|
@@ -296,7 +296,8 @@ module Pgbus
|
|
|
296
296
|
last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
|
|
297
297
|
headers = DeadLetterHeader.build(
|
|
298
298
|
existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "consumer",
|
|
299
|
-
source_queue: config.queue_name(queue_name),
|
|
299
|
+
source_queue: config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
|
|
300
|
+
source_enqueued_at: message.enqueued_at, attempts: message.read_ct.to_i,
|
|
300
301
|
max_retries: config.max_retries, error: last_error
|
|
301
302
|
)
|
|
302
303
|
Pgbus.client.move_to_dead_letter(queue_name, message, headers: headers)
|
|
@@ -505,12 +505,22 @@ module Pgbus
|
|
|
505
505
|
# Unbound rows (synthetic "pending" queue and/or msg_id=0) must not probe
|
|
506
506
|
# pgmq.q_<prefix>_pending — that table does not exist, message_exists?
|
|
507
507
|
# returns nil, and the reaper would keep the lock forever (issue #418).
|
|
508
|
-
# Scan every live queue for the payload key instead.
|
|
508
|
+
# Scan every live queue for the payload key instead. A job parked behind
|
|
509
|
+
# its concurrency slot is in no queue: its key counts as live while it
|
|
510
|
+
# is parked (issue #564). The parked jobs are asked first: a promotion
|
|
511
|
+
# sends the job's message before its parked row's delete commits (or
|
|
512
|
+
# both in one transaction), so a job promoted between the two reads is
|
|
513
|
+
# still parked for the first or already queued for the second. Read the
|
|
514
|
+
# other way round it could be in neither.
|
|
509
515
|
def gone_unbound_locks(unbound)
|
|
510
516
|
return [] if unbound.empty?
|
|
511
517
|
|
|
512
|
-
|
|
513
|
-
unbound.reject { |key|
|
|
518
|
+
parked = BlockedExecution.parked_uniqueness_keys(unbound.map(&:lock_key))
|
|
519
|
+
unparked = unbound.reject { |key| parked.include?(key.lock_key) }
|
|
520
|
+
return [] if unparked.empty?
|
|
521
|
+
|
|
522
|
+
present = Pgbus.client.uniqueness_keys_present(unparked.map(&:lock_key))
|
|
523
|
+
unparked.reject { |key| present.include?(key.lock_key) }
|
|
514
524
|
end
|
|
515
525
|
|
|
516
526
|
# Returns true if the message referenced by this bound lock is definitely
|
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -407,12 +407,21 @@ module Pgbus
|
|
|
407
407
|
# so we also pass `limit: qty` to cap the total across all queues —
|
|
408
408
|
# otherwise we get `queue_count * qty` messages and overflow the
|
|
409
409
|
# execution pool, crashing the worker fork (issue #123).
|
|
410
|
+
#
|
|
411
|
+
# The logical name is the configured one the table was made from, not
|
|
412
|
+
# the table name with its prefix cut: a name normalized for its table
|
|
413
|
+
# ("orders-handler" → "orders_handler") would otherwise reach the
|
|
414
|
+
# executor spelled unlike the name its jobs' uniqueness locks are bound
|
|
415
|
+
# under, and unlike every other fetch path (issue #564).
|
|
410
416
|
def fetch_multi(active_queues, qty)
|
|
411
417
|
messages = Pgbus.client.read_multi(active_queues, qty: qty, limit: qty) || []
|
|
418
|
+
return messages if messages.empty?
|
|
419
|
+
|
|
412
420
|
prefix = "#{config.queue_prefix}_"
|
|
421
|
+
configured = active_queues.to_h { |q| [config.queue_name(q), q] }
|
|
413
422
|
|
|
414
423
|
messages.map do |m|
|
|
415
|
-
logical = m.queue_name&.delete_prefix(prefix) || active_queues.first
|
|
424
|
+
logical = configured[m.queue_name] || m.queue_name&.delete_prefix(prefix) || active_queues.first
|
|
416
425
|
[logical, m]
|
|
417
426
|
end
|
|
418
427
|
end
|
|
@@ -143,7 +143,7 @@ module Pgbus
|
|
|
143
143
|
key = resolve_uniqueness_key(job_class, config, task)
|
|
144
144
|
return nil unless key
|
|
145
145
|
|
|
146
|
-
acquired = UniquenessKey.acquire!(key, queue_name: resolve_queue(task), msg_id: 0)
|
|
146
|
+
acquired = UniquenessKey.acquire!(key, queue_name: Uniqueness.lock_queue_name(resolve_queue(task)), msg_id: 0)
|
|
147
147
|
|
|
148
148
|
if acquired
|
|
149
149
|
key
|
|
@@ -72,6 +72,7 @@ module Pgbus
|
|
|
72
72
|
@schema_mutex = Mutex.new
|
|
73
73
|
@seats_supported = nil
|
|
74
74
|
@probed_at = nil
|
|
75
|
+
@schema_generation = 0
|
|
75
76
|
|
|
76
77
|
class << self
|
|
77
78
|
# True when pgbus_presence_members has the seat columns (the
|
|
@@ -81,17 +82,29 @@ module Pgbus
|
|
|
81
82
|
# The answer is read and written under @schema_mutex only (issue
|
|
82
83
|
# #527). A successful probe is kept as above; a probe that raises is
|
|
83
84
|
# not, so the next call probes again.
|
|
85
|
+
#
|
|
86
|
+
# The probe itself runs outside @schema_mutex (issue #566): it waits
|
|
87
|
+
# for the adapter's lock, and a request inside a transaction holds
|
|
88
|
+
# that lock while it asks seats_supported?. Two threads may probe at
|
|
89
|
+
# once; a true answer one of them stored is never taken back, and a
|
|
90
|
+
# probe that a reset_schema_cache! overtook stores nothing.
|
|
84
91
|
def seats_supported?
|
|
92
|
+
cached, generation = @schema_mutex.synchronize do
|
|
93
|
+
[(@seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)), @schema_generation]
|
|
94
|
+
end
|
|
95
|
+
return cached unless cached.nil?
|
|
96
|
+
|
|
97
|
+
rows = exec_params(<<~SQL, [])
|
|
98
|
+
SELECT 1 AS present FROM information_schema.columns
|
|
99
|
+
WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
|
|
100
|
+
AND table_schema = ANY (current_schemas(false))
|
|
101
|
+
SQL
|
|
102
|
+
supported = !rows.to_a.empty?
|
|
85
103
|
@schema_mutex.synchronize do
|
|
86
|
-
next
|
|
104
|
+
next supported unless generation == @schema_generation
|
|
87
105
|
|
|
88
|
-
rows = connection.exec_params(<<~SQL, [])
|
|
89
|
-
SELECT 1 AS present FROM information_schema.columns
|
|
90
|
-
WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
|
|
91
|
-
AND table_schema = ANY (current_schemas(false))
|
|
92
|
-
SQL
|
|
93
106
|
@probed_at = monotonic_now
|
|
94
|
-
@seats_supported
|
|
107
|
+
@seats_supported ||= supported
|
|
95
108
|
end
|
|
96
109
|
end
|
|
97
110
|
|
|
@@ -108,6 +121,7 @@ module Pgbus
|
|
|
108
121
|
|
|
109
122
|
def reset_schema_cache!
|
|
110
123
|
@schema_mutex.synchronize do
|
|
124
|
+
@schema_generation += 1
|
|
111
125
|
@probed_at = nil
|
|
112
126
|
@seats_supported = nil
|
|
113
127
|
end
|
|
@@ -124,7 +138,7 @@ module Pgbus
|
|
|
124
138
|
def expire_seats!(ttl_for:, min_ttl:)
|
|
125
139
|
return [] unless seats_supported?
|
|
126
140
|
|
|
127
|
-
candidates =
|
|
141
|
+
candidates = exec_params(<<~SQL, [min_ttl])
|
|
128
142
|
SELECT stream_name, connection_id FROM pgbus_presence_members
|
|
129
143
|
WHERE connection_id IS NOT NULL AND last_seen_at < NOW() - make_interval(secs => $1)
|
|
130
144
|
SQL
|
|
@@ -133,23 +147,23 @@ module Pgbus
|
|
|
133
147
|
end
|
|
134
148
|
end
|
|
135
149
|
|
|
150
|
+
# The PG::Connection under the adapter. Run SQL through exec_params,
|
|
151
|
+
# not on this: the adapter's lock is released once it is returned.
|
|
136
152
|
def connection
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
# live in a separate database accessed via Pgbus::BusRecord;
|
|
140
|
-
# the default path uses ActiveRecord::Base. Matches the
|
|
141
|
-
# canonical pattern in Pgbus::Process::QueueLock#connection
|
|
142
|
-
# and Pgbus::Configuration's connection probe.
|
|
143
|
-
unless defined?(::ActiveRecord::Base)
|
|
144
|
-
raise Pgbus::ConfigurationError,
|
|
145
|
-
"Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
|
|
146
|
-
end
|
|
153
|
+
adapter.raw_connection
|
|
154
|
+
end
|
|
147
155
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
156
|
+
# Runs one statement on the adapter's PG::Connection while holding
|
|
157
|
+
# the adapter's lock (issue #566). raw_connection takes that lock
|
|
158
|
+
# only to hand the connection out; on a pinned pool (transactional
|
|
159
|
+
# tests) every thread shares the one adapter, and its lock is all
|
|
160
|
+
# that keeps a second thread's statement off the libpq connection
|
|
161
|
+
# while this one is in flight. Unpinned, the lock is a NullLock. The
|
|
162
|
+
# lock is reentrant, so a thread already holding it (a request
|
|
163
|
+
# inside a transaction) re-enters it.
|
|
164
|
+
def exec_params(sql, binds)
|
|
165
|
+
adapter = self.adapter
|
|
166
|
+
adapter.lock.synchronize { adapter.raw_connection.exec_params(sql, binds) }
|
|
153
167
|
end
|
|
154
168
|
|
|
155
169
|
# Reports a seat join/leave to streams_presence_on_change (issue #515).
|
|
@@ -171,6 +185,21 @@ module Pgbus
|
|
|
171
185
|
|
|
172
186
|
private
|
|
173
187
|
|
|
188
|
+
# Respect multi-database setups (connects_to). When
|
|
189
|
+
# Pgbus.configuration.connects_to is set, the pgbus tables
|
|
190
|
+
# live in a separate database accessed via Pgbus::BusRecord;
|
|
191
|
+
# the default path uses ActiveRecord::Base. Matches the
|
|
192
|
+
# canonical pattern in Pgbus::Process::QueueLock#connection
|
|
193
|
+
# and Pgbus::Configuration's connection probe.
|
|
194
|
+
def adapter
|
|
195
|
+
unless defined?(::ActiveRecord::Base)
|
|
196
|
+
raise Pgbus::ConfigurationError,
|
|
197
|
+
"Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ::ActiveRecord::Base.connection
|
|
201
|
+
end
|
|
202
|
+
|
|
174
203
|
def schema_answer_fresh?(supported, probed_at)
|
|
175
204
|
return false if supported.nil? || probed_at.nil?
|
|
176
205
|
|
|
@@ -178,7 +207,7 @@ module Pgbus
|
|
|
178
207
|
end
|
|
179
208
|
|
|
180
209
|
def expire_stream_seats(stream_name, connection_ids, ttl)
|
|
181
|
-
rows =
|
|
210
|
+
rows = exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
|
|
182
211
|
DELETE FROM pgbus_presence_members
|
|
183
212
|
WHERE stream_name = $1 AND connection_id = ANY($2::text[])
|
|
184
213
|
AND last_seen_at < NOW() - make_interval(secs => $3)
|
|
@@ -306,7 +335,7 @@ module Pgbus
|
|
|
306
335
|
return touch_seats(Array(connection_id)) if connection_id
|
|
307
336
|
|
|
308
337
|
member_id = member_id.to_s
|
|
309
|
-
|
|
338
|
+
exec_params(<<~SQL, [@stream.name, member_id])
|
|
310
339
|
UPDATE pgbus_presence_members
|
|
311
340
|
SET last_seen_at = NOW()
|
|
312
341
|
WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
|
|
@@ -320,7 +349,7 @@ module Pgbus
|
|
|
320
349
|
def set_state(connection_id:, state:)
|
|
321
350
|
raise ArgumentError, "set_state takes a Hash (got #{state.class})" unless state.is_a?(Hash)
|
|
322
351
|
|
|
323
|
-
rows =
|
|
352
|
+
rows = exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
|
|
324
353
|
UPDATE pgbus_presence_members
|
|
325
354
|
SET state = state || $3::jsonb, last_seen_at = NOW()
|
|
326
355
|
WHERE stream_name = $1 AND connection_id = $2
|
|
@@ -335,7 +364,7 @@ module Pgbus
|
|
|
335
364
|
# table each row is one seat and also carries connection_id, seat
|
|
336
365
|
# and state; group by "id" for one entry per member.
|
|
337
366
|
def members
|
|
338
|
-
rows =
|
|
367
|
+
rows = exec_params(<<~SQL, [@stream.name])
|
|
339
368
|
SELECT #{seats? ? SEAT_RECORD_COLUMNS : RECORD_COLUMNS}
|
|
340
369
|
FROM pgbus_presence_members
|
|
341
370
|
WHERE stream_name = $1
|
|
@@ -347,7 +376,7 @@ module Pgbus
|
|
|
347
376
|
# Returns the count of current members. Faster than members.size
|
|
348
377
|
# because it doesn't deserialize metadata.
|
|
349
378
|
def count
|
|
350
|
-
rows =
|
|
379
|
+
rows = exec_params(<<~SQL, [@stream.name])
|
|
351
380
|
SELECT COUNT(*) AS n FROM pgbus_presence_members WHERE stream_name = $1
|
|
352
381
|
SQL
|
|
353
382
|
rows.first["n"].to_i
|
|
@@ -362,7 +391,7 @@ module Pgbus
|
|
|
362
391
|
# workers running the sweep concurrently won't double-emit leave
|
|
363
392
|
# events. Yields each expired record when a block is given.
|
|
364
393
|
def sweep!(older_than:)
|
|
365
|
-
rows =
|
|
394
|
+
rows = exec_params(<<~SQL, [@stream.name, older_than])
|
|
366
395
|
DELETE FROM pgbus_presence_members
|
|
367
396
|
WHERE stream_name = $1 AND last_seen_at < $2#{" AND connection_id IS NULL" if seats_now?}
|
|
368
397
|
RETURNING #{RECORD_COLUMNS}
|
|
@@ -388,7 +417,7 @@ module Pgbus
|
|
|
388
417
|
|
|
389
418
|
def upsert_member_once(member_id, metadata)
|
|
390
419
|
conflict = seats? ? "(stream_name, member_id) WHERE connection_id IS NULL" : "(stream_name, member_id)"
|
|
391
|
-
rows =
|
|
420
|
+
rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
|
|
392
421
|
INSERT INTO pgbus_presence_members (stream_name, member_id, metadata, joined_at, last_seen_at)
|
|
393
422
|
VALUES ($1, $2, $3::jsonb, NOW(), NOW())
|
|
394
423
|
ON CONFLICT #{conflict}
|
|
@@ -399,7 +428,7 @@ module Pgbus
|
|
|
399
428
|
end
|
|
400
429
|
|
|
401
430
|
def upsert_seat(member_id, metadata, connection_id, seat)
|
|
402
|
-
rows =
|
|
431
|
+
rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
|
|
403
432
|
INSERT INTO pgbus_presence_members
|
|
404
433
|
(stream_name, member_id, metadata, connection_id, seat, joined_at, last_seen_at)
|
|
405
434
|
VALUES ($1, $2, $3::jsonb, $4, $5, NOW(), NOW())
|
|
@@ -412,7 +441,7 @@ module Pgbus
|
|
|
412
441
|
end
|
|
413
442
|
|
|
414
443
|
def delete_member(member_id)
|
|
415
|
-
rows =
|
|
444
|
+
rows = exec_params(<<~SQL, [@stream.name, member_id])
|
|
416
445
|
DELETE FROM pgbus_presence_members
|
|
417
446
|
WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
|
|
418
447
|
RETURNING #{RECORD_COLUMNS}
|
|
@@ -422,7 +451,7 @@ module Pgbus
|
|
|
422
451
|
end
|
|
423
452
|
|
|
424
453
|
def delete_seat(connection_id)
|
|
425
|
-
rows =
|
|
454
|
+
rows = exec_params(<<~SQL, [@stream.name, connection_id])
|
|
426
455
|
DELETE FROM pgbus_presence_members
|
|
427
456
|
WHERE stream_name = $1 AND connection_id = $2
|
|
428
457
|
RETURNING #{SEAT_RECORD_COLUMNS}
|
|
@@ -434,7 +463,7 @@ module Pgbus
|
|
|
434
463
|
def touch_seats(connection_ids)
|
|
435
464
|
return [] if connection_ids.empty?
|
|
436
465
|
|
|
437
|
-
rows =
|
|
466
|
+
rows = exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
|
|
438
467
|
UPDATE pgbus_presence_members
|
|
439
468
|
SET last_seen_at = NOW()
|
|
440
469
|
WHERE stream_name = $1 AND connection_id = ANY($2::text[])
|
|
@@ -443,8 +472,8 @@ module Pgbus
|
|
|
443
472
|
rows.to_a.map { |row| row["connection_id"] }
|
|
444
473
|
end
|
|
445
474
|
|
|
446
|
-
def
|
|
447
|
-
self.class.
|
|
475
|
+
def exec_params(sql, binds)
|
|
476
|
+
self.class.exec_params(sql, binds)
|
|
448
477
|
end
|
|
449
478
|
end
|
|
450
479
|
end
|