pgbus 0.22.0 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/README.md +41 -14
- data/Rakefile +7 -1
- data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
- data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
- data/app/controllers/pgbus/jobs_controller.rb +2 -1
- data/app/models/pgbus/blocked_execution.rb +3 -2
- data/app/models/pgbus/queue_state.rb +6 -0
- data/app/models/pgbus/uniqueness_key.rb +11 -0
- data/config/locales/da.yml +1 -0
- data/config/locales/de.yml +1 -0
- data/config/locales/en.yml +1 -0
- data/config/locales/es.yml +1 -0
- data/config/locales/fi.yml +1 -0
- data/config/locales/fr.yml +1 -0
- data/config/locales/it.yml +1 -0
- data/config/locales/ja.yml +1 -0
- data/config/locales/nb.yml +1 -0
- data/config/locales/nl.yml +1 -0
- data/config/locales/pt.yml +1 -0
- data/config/locales/sv.yml +1 -0
- data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
- data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
- data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
- data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
- data/lib/pgbus/active_job/adapter.rb +3 -2
- data/lib/pgbus/active_job/executor.rb +142 -20
- data/lib/pgbus/active_job/quiet_logs.rb +57 -0
- data/lib/pgbus/autovacuum_tuning.rb +2 -2
- data/lib/pgbus/cli/dlq.rb +2 -1
- data/lib/pgbus/client/async_commit.rb +149 -0
- data/lib/pgbus/client/native_statements.rb +28 -16
- data/lib/pgbus/client.rb +181 -17
- data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
- data/lib/pgbus/configuration.rb +71 -6
- data/lib/pgbus/dead_letter_header.rb +3 -0
- data/lib/pgbus/doctor.rb +55 -2
- data/lib/pgbus/failed_event_recorder.rb +5 -2
- data/lib/pgbus/generators/migration_detector.rb +29 -2
- data/lib/pgbus/instrumentation.rb +16 -1
- data/lib/pgbus/metrics/subscriber.rb +27 -1
- data/lib/pgbus/process/completer.rb +4 -2
- data/lib/pgbus/process/worker.rb +28 -3
- data/lib/pgbus/queue_name_fold.rb +145 -0
- data/lib/pgbus/queue_name_validator.rb +20 -2
- data/lib/pgbus/recurring/schedule.rb +22 -13
- data/lib/pgbus/stat_buffer.rb +67 -26
- data/lib/pgbus/streams/cursor.rb +17 -6
- data/lib/pgbus/streams/envelope.rb +16 -0
- data/lib/pgbus/streams/key.rb +1 -1
- data/lib/pgbus/table_maintenance.rb +1 -1
- data/lib/pgbus/uniqueness.rb +43 -16
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
- data/lib/pgbus/web/data_source/resend.rb +185 -0
- data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
- data/lib/pgbus/web/data_source.rb +55 -37
- data/lib/pgbus/web/dead_letter_reason.rb +6 -0
- data/lib/pgbus/web/streamer/connection.rb +36 -29
- data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
- data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
- data/lib/pgbus/web/streamer/instance.rb +8 -5
- data/lib/pgbus/web/streamer/listener.rb +239 -62
- data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
- data/lib/pgbus/web/streamer/registry.rb +55 -29
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
- data/lib/tasks/pgbus_queues.rake +27 -0
- data/sig/pgbus/active_job/adapter.rbs +1 -1
- data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
- data/sig/pgbus/client.rbs +6 -5
- data/sig/pgbus/configuration.rbs +4 -0
- metadata +11 -3
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -52,6 +52,37 @@ module Pgbus
|
|
|
52
52
|
# Batch::Sweep then cannot tell a deleted batch job from one whose message
|
|
53
53
|
# vanished another way (a purged queue): both count as completed, silently.
|
|
54
54
|
attr_reader :completed_jobs
|
|
55
|
+
# completion_synchronous_commit (issue #619, opt-in): false commits a job's
|
|
56
|
+
# completion (the executor's archive or delete, the completer's
|
|
57
|
+
# archive_batch or delete_batch) with `SET LOCAL synchronous_commit TO off`,
|
|
58
|
+
# in one libpq pipeline: BEGIN, the SET LOCAL, the prepared statement and
|
|
59
|
+
# COMMIT go out with one sync, so the job stops waiting for its WAL flush.
|
|
60
|
+
# A completion is then acknowledged before its WAL record reaches disk: a
|
|
61
|
+
# Postgres server crash (not a client crash, not a planned switchover), or a
|
|
62
|
+
# failover to a standby that has not yet received that WAL (it also stops
|
|
63
|
+
# waiting for a synchronous standby), within about three times
|
|
64
|
+
# wal_writer_delay can lose completions committed in that window, and each
|
|
65
|
+
# lost completion redelivers its job after visibility_timeout. Sends, retries, dead-letter moves, the EventBus
|
|
66
|
+
# consumer's archive and dashboard actions keep full durability, and a
|
|
67
|
+
# completion inside a caller's transaction is left to that transaction.
|
|
68
|
+
# Native statements on pgbus's own connections only: with
|
|
69
|
+
# native_statements = false, or on ActiveRecord's shared connection (a
|
|
70
|
+
# connection_params Proc), it has no effect (pgbus doctor warns). Needs
|
|
71
|
+
# libpq 14+ and pg 1.5+ (pipeline mode), checked by validate!. true
|
|
72
|
+
# (default) commits as before.
|
|
73
|
+
attr_reader :completion_synchronous_commit
|
|
74
|
+
# quiet_active_job_logs (issue #618): true makes each worker process
|
|
75
|
+
# unsubscribe ActiveJob's LogSubscriber from perform_start.active_job and
|
|
76
|
+
# perform.active_job (Pgbus::ActiveJob::QuietLogs), so the per-job
|
|
77
|
+
# "Performing …" / "Performed …" info lines stop, and with them ActiveJob's
|
|
78
|
+
# "Error performing …" line (the same perform listener writes it): an
|
|
79
|
+
# exception that escapes ActiveJob's handlers is still logged by
|
|
80
|
+
# ErrorReporter and recorded in pgbus_failed_events, one retry_on catches
|
|
81
|
+
# still logs ActiveJob's "Retrying …" line, and a job a before_perform
|
|
82
|
+
# callback halted logs nothing.
|
|
83
|
+
# Enqueued, Retrying, Stopped retrying and Discarded still log, and the web
|
|
84
|
+
# process is untouched. Without ActiveJob it does nothing.
|
|
85
|
+
attr_reader :quiet_active_job_logs
|
|
55
86
|
# visibility_heartbeat / visibility_heartbeat_interval: while a job runs,
|
|
56
87
|
# its message's visibility timeout is re-armed every interval seconds
|
|
57
88
|
# (default: a third of visibility_timeout), so a job that outlives the
|
|
@@ -367,6 +398,8 @@ module Pgbus
|
|
|
367
398
|
@completion_batching = false
|
|
368
399
|
@completion_batch_size = 100
|
|
369
400
|
@completed_jobs = :archive
|
|
401
|
+
@completion_synchronous_commit = true
|
|
402
|
+
@quiet_active_job_logs = false
|
|
370
403
|
@execution_mode = :threads
|
|
371
404
|
|
|
372
405
|
@max_jobs_per_worker = nil
|
|
@@ -564,12 +597,13 @@ module Pgbus
|
|
|
564
597
|
@streams_idle_timeout = 3_600 # 1 hour
|
|
565
598
|
# 250ms — this value plays two roles: (1) the TCP keepalive
|
|
566
599
|
# interval for the streamer's PG LISTEN connection, and (2) the
|
|
567
|
-
#
|
|
568
|
-
#
|
|
569
|
-
#
|
|
570
|
-
#
|
|
571
|
-
#
|
|
572
|
-
#
|
|
600
|
+
# budget (plus 1s) a stream's first connect waits for a stuck local
|
|
601
|
+
# Listener to acknowledge its ensure_listening (with
|
|
602
|
+
# streams_listen_scope :master, HubClient waits this cycle plus 2s
|
|
603
|
+
# before failing over). A live listener
|
|
604
|
+
# is woken through a pipe and acks within a thread hop, and a
|
|
605
|
+
# connect on a stream that already has a subscriber does not
|
|
606
|
+
# wait at all (issue #630), so connects no longer pay this cycle.
|
|
573
607
|
@streams_listen_health_check_ms = 250
|
|
574
608
|
@streams_write_deadline_ms = 5_000
|
|
575
609
|
# Deadline (ms) for a socket write made INSIDE the dispatcher's
|
|
@@ -856,6 +890,22 @@ module Pgbus
|
|
|
856
890
|
@completed_jobs = mode
|
|
857
891
|
end
|
|
858
892
|
|
|
893
|
+
def completion_synchronous_commit=(value)
|
|
894
|
+
unless [true, false].include?(value)
|
|
895
|
+
raise Pgbus::ConfigurationError, "completion_synchronous_commit must be true or false (got #{value.inspect})"
|
|
896
|
+
end
|
|
897
|
+
|
|
898
|
+
@completion_synchronous_commit = value
|
|
899
|
+
end
|
|
900
|
+
|
|
901
|
+
def quiet_active_job_logs=(value)
|
|
902
|
+
unless [true, false].include?(value)
|
|
903
|
+
raise Pgbus::ConfigurationError, "quiet_active_job_logs must be true or false (got #{value.inspect})"
|
|
904
|
+
end
|
|
905
|
+
|
|
906
|
+
@quiet_active_job_logs = value
|
|
907
|
+
end
|
|
908
|
+
|
|
859
909
|
def native_statements=(value)
|
|
860
910
|
raise Pgbus::ConfigurationError, "native_statements must be true or false" unless [true, false].include?(value)
|
|
861
911
|
|
|
@@ -1052,6 +1102,7 @@ module Pgbus
|
|
|
1052
1102
|
|
|
1053
1103
|
validate_read_ahead!
|
|
1054
1104
|
validate_completion_batching!
|
|
1105
|
+
validate_completion_synchronous_commit!
|
|
1055
1106
|
validate_capsule_processes!
|
|
1056
1107
|
|
|
1057
1108
|
if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
|
|
@@ -1843,6 +1894,20 @@ module Pgbus
|
|
|
1843
1894
|
"execution_mode :async"
|
|
1844
1895
|
end
|
|
1845
1896
|
|
|
1897
|
+
# The asynchronous completion commit runs in libpq pipeline mode (issue
|
|
1898
|
+
# #619): libpq 14+ and a pg gem that wraps it (1.5+). Checked here, once,
|
|
1899
|
+
# rather than on every completion.
|
|
1900
|
+
def validate_completion_synchronous_commit!
|
|
1901
|
+
return if completion_synchronous_commit
|
|
1902
|
+
|
|
1903
|
+
require "pg" unless defined?(::PG::Connection)
|
|
1904
|
+
return if ::PG::Connection.method_defined?(:enter_pipeline_mode) && ::PG.library_version >= 140_000
|
|
1905
|
+
|
|
1906
|
+
raise Pgbus::ConfigurationError,
|
|
1907
|
+
"completion_synchronous_commit = false needs libpq 14+ and pg 1.5+ (pipeline mode); " \
|
|
1908
|
+
"this process has libpq #{::PG.library_version} and pg #{::PG::VERSION}"
|
|
1909
|
+
end
|
|
1910
|
+
|
|
1846
1911
|
# Shared resolver for a dedicated connection (streamer LISTEN / worker
|
|
1847
1912
|
# NotifyListener) that can override the base `connection_options` to bypass a
|
|
1848
1913
|
# pooler. Precedence: a full `url` wins outright; otherwise `host`/`port`
|
|
@@ -20,6 +20,9 @@ module Pgbus
|
|
|
20
20
|
ORIGINAL_KEY = "pgbus_original_headers"
|
|
21
21
|
VERSION = 1
|
|
22
22
|
REASON_MAX_RETRIES = "max_retries_exceeded"
|
|
23
|
+
# A :while_executing duplicate skipped on every read while another
|
|
24
|
+
# message held its key, until read_ct passed max_retries (issue #629).
|
|
25
|
+
REASON_UNIQUENESS_CONFLICT = "uniqueness_conflict"
|
|
23
26
|
# The release that started writing the block: rows without it predate it.
|
|
24
27
|
SINCE = "0.18.0"
|
|
25
28
|
MESSAGE_LIMIT = 1_000
|
data/lib/pgbus/doctor.rb
CHANGED
|
@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
|
|
|
5
5
|
|
|
6
6
|
module Pgbus
|
|
7
7
|
# Preflight diagnostics for a pgbus deployment — the single command that
|
|
8
|
-
# answers "is this environment healthy enough to run?". Runs
|
|
8
|
+
# answers "is this environment healthy enough to run?". Runs fourteen checks and
|
|
9
9
|
# returns a machine-readable result plus a human report, so `pgbus doctor`
|
|
10
10
|
# and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
|
|
11
11
|
# 1 on any failure).
|
|
@@ -36,13 +36,15 @@ module Pgbus
|
|
|
36
36
|
"Database" => :check_database,
|
|
37
37
|
"PGMQ schema" => :check_pgmq_schema,
|
|
38
38
|
"Queues" => :check_queues,
|
|
39
|
+
"Queue name case" => :check_queue_name_case,
|
|
39
40
|
"LISTEN/NOTIFY" => :check_notify,
|
|
40
41
|
"Process liveness" => :check_processes,
|
|
41
42
|
"GlobalID allowlist" => :check_allowed_global_id_models,
|
|
42
43
|
"Broadcast queue" => :check_broadcast_queue,
|
|
43
44
|
"Primary affinity" => :check_primary,
|
|
44
45
|
"Dedicated connections" => :check_dedicated_connections,
|
|
45
|
-
"Connection budget" => :check_connection_budget
|
|
46
|
+
"Connection budget" => :check_connection_budget,
|
|
47
|
+
"Completion commit" => :check_completion_commit
|
|
46
48
|
}.freeze
|
|
47
49
|
|
|
48
50
|
# Process liveness reads the pgbus_processes table (via HealthAnalyzer), so
|
|
@@ -128,6 +130,7 @@ module Pgbus
|
|
|
128
130
|
resolved_pool_size: safe { @config.resolved_pool_size },
|
|
129
131
|
listen_notify: @config.listen_notify,
|
|
130
132
|
roles: @config.roles || "all",
|
|
133
|
+
completion_commit: @config.completion_synchronous_commit ? "sync" : "async",
|
|
131
134
|
capsules: capsule_summary,
|
|
132
135
|
database_url: redacted_database_url,
|
|
133
136
|
connection_params: redacted_connection_params
|
|
@@ -252,6 +255,30 @@ module Pgbus
|
|
|
252
255
|
Check.new(name: "Queues", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
253
256
|
end
|
|
254
257
|
|
|
258
|
+
# How many mixed-case queue names the queue-name case check lists.
|
|
259
|
+
CASE_NAMES_SHOWN = 5
|
|
260
|
+
private_constant :CASE_NAMES_SHOWN
|
|
261
|
+
|
|
262
|
+
# 4b. Queue-name case (issue #633). PGMQ spells every queue table in lower
|
|
263
|
+
# case, and pgbus now spells every queue name that way; a pgmq.meta row an
|
|
264
|
+
# older pgbus created as "pgbus_Orders" shares pgmq.q_pgbus_orders with the
|
|
265
|
+
# current spelling. Its messages are reachable, so this warns rather than
|
|
266
|
+
# fails: the dashboard lists the queue twice until the fold migration runs.
|
|
267
|
+
def check_queue_name_case
|
|
268
|
+
names = @client.mixed_case_queue_names
|
|
269
|
+
return Check.new(name: "Queue name case", status: :ok, detail: "every queue is spelled in lower case") if names.empty?
|
|
270
|
+
|
|
271
|
+
shown = names.first(CASE_NAMES_SHOWN).join(", ")
|
|
272
|
+
shown += ", …" if names.size > CASE_NAMES_SHOWN
|
|
273
|
+
Check.new(name: "Queue name case", status: :warn,
|
|
274
|
+
detail: "#{names.size} queue(s) in pgmq.meta spelled in mixed case (#{shown}); their tables are " \
|
|
275
|
+
"lower case and pgbus now uses that spelling — once every process runs this pgbus, run " \
|
|
276
|
+
"`rails generate pgbus:fold_queue_name_case` and migrate " \
|
|
277
|
+
"(`rake pgbus:queues:fold_case` once that migration has run)")
|
|
278
|
+
rescue StandardError => e
|
|
279
|
+
Check.new(name: "Queue name case", status: :warn, detail: "could not read pgmq.meta (#{e.class}: #{e.message})")
|
|
280
|
+
end
|
|
281
|
+
|
|
255
282
|
# 5. LISTEN/NOTIFY — when configured on, every configured queue should carry
|
|
256
283
|
# the insert-NOTIFY trigger. A missing trigger is a warning, not a failure:
|
|
257
284
|
# pgbus still works (it falls back to polling), it just loses instant wakeup.
|
|
@@ -395,6 +422,32 @@ module Pgbus
|
|
|
395
422
|
Check.new(name: "Dedicated connections", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
396
423
|
end
|
|
397
424
|
|
|
425
|
+
# config.completion_synchronous_commit (issue #619): informational, except
|
|
426
|
+
# that false does nothing with native_statements off or on ActiveRecord's
|
|
427
|
+
# shared connection, which warns.
|
|
428
|
+
def check_completion_commit
|
|
429
|
+
return Check.new(name: "Completion commit", status: :ok, detail: "synchronous (default)") if
|
|
430
|
+
@config.completion_synchronous_commit
|
|
431
|
+
|
|
432
|
+
unless @config.native_statements
|
|
433
|
+
return Check.new(name: "Completion commit", status: :warn,
|
|
434
|
+
detail: "completion_synchronous_commit = false has no effect: completions run through " \
|
|
435
|
+
"pgmq's functions (native_statements = false)")
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
if @client.respond_to?(:shared_connection?) && @client.shared_connection?
|
|
439
|
+
return Check.new(name: "Completion commit", status: :warn,
|
|
440
|
+
detail: "completion_synchronous_commit = false has no effect: pgbus shares ActiveRecord's " \
|
|
441
|
+
"connection (connection_params Proc); give pgbus its own connections to use it")
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
Check.new(name: "Completion commit", status: :ok,
|
|
445
|
+
detail: "asynchronous: completions commit with synchronous_commit off; a Postgres server crash can " \
|
|
446
|
+
"lose the last completions, whose jobs run again after visibility_timeout")
|
|
447
|
+
rescue StandardError => e
|
|
448
|
+
Check.new(name: "Completion commit", status: :warn, detail: "#{e.class}: #{e.message}")
|
|
449
|
+
end
|
|
450
|
+
|
|
398
451
|
# How many direct LISTEN connections this config pins at steady state
|
|
399
452
|
# (issue #381) — informational (always :ok) so operators can do capacity
|
|
400
453
|
# math on the pooler's direct-connection budget from the doctor output.
|
|
@@ -85,11 +85,14 @@ module Pgbus
|
|
|
85
85
|
# queue is dropped: PGMQ restarts msg_ids on recreate, so a surviving
|
|
86
86
|
# row would aim the dashboard's retry/discard at an unrelated message.
|
|
87
87
|
# Queue names are validated word characters, so the array literal is safe.
|
|
88
|
+
# Matched in any case (issue #633): the dropped queue's name is lower
|
|
89
|
+
# case, while a row may carry the executor's configured spelling
|
|
90
|
+
# ("Critical") or a physical name from before #633.
|
|
88
91
|
def clear_queue!(queue_names)
|
|
89
92
|
connection.exec_delete(
|
|
90
|
-
"DELETE FROM pgbus_failed_events WHERE queue_name = ANY($1::text[])",
|
|
93
|
+
"DELETE FROM pgbus_failed_events WHERE lower(queue_name) = ANY($1::text[])",
|
|
91
94
|
"FailedEvent Clear Queue",
|
|
92
|
-
["{#{queue_names.join(",")}}"]
|
|
95
|
+
["{#{queue_names.map(&:downcase).join(",")}}"]
|
|
93
96
|
)
|
|
94
97
|
rescue StandardError => e
|
|
95
98
|
Pgbus.logger.error do
|
|
@@ -102,7 +102,8 @@ module Pgbus
|
|
|
102
102
|
add_batch_executions: "pgbus:add_batch_executions",
|
|
103
103
|
add_batch_callback_jobs: "pgbus:add_batch_callback_jobs",
|
|
104
104
|
tune_autovacuum: "pgbus:tune_autovacuum",
|
|
105
|
-
tune_fillfactor: "pgbus:tune_fillfactor"
|
|
105
|
+
tune_fillfactor: "pgbus:tune_fillfactor",
|
|
106
|
+
fold_queue_name_case: "pgbus:fold_queue_name_case"
|
|
106
107
|
}.freeze
|
|
107
108
|
|
|
108
109
|
# Human-friendly description of each migration for the generator
|
|
@@ -128,9 +129,15 @@ module Pgbus
|
|
|
128
129
|
add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
|
|
129
130
|
add_batch_callback_jobs: "batch callback-job columns (configured ActiveJob instances as batch callbacks)",
|
|
130
131
|
tune_autovacuum: "autovacuum tuning for PGMQ queue and archive tables",
|
|
131
|
-
tune_fillfactor: "fillfactor=70 on PGMQ queue tables (reduces page density during update churn)"
|
|
132
|
+
tune_fillfactor: "fillfactor=70 on PGMQ queue tables (reduces page density during update churn)",
|
|
133
|
+
fold_queue_name_case: "lower-case spelling for queues created in mixed case (PGMQ's table names are lower case)"
|
|
132
134
|
}.freeze
|
|
133
135
|
|
|
136
|
+
# A pgbus pgmq.meta row ($1: the lower-case "<queue_prefix>_") spelled
|
|
137
|
+
# with an upper-case letter (issue #633).
|
|
138
|
+
MIXED_CASE_QUEUE_SQL = "SELECT 1 FROM pgmq.meta WHERE queue_name <> lower(queue_name) " \
|
|
139
|
+
"AND left(lower(queue_name), length($1)) = $1 LIMIT 1"
|
|
140
|
+
|
|
134
141
|
def initialize(connection)
|
|
135
142
|
@connection = connection
|
|
136
143
|
end
|
|
@@ -143,6 +150,9 @@ module Pgbus
|
|
|
143
150
|
return [FRESH_INSTALL] if fresh_install?
|
|
144
151
|
|
|
145
152
|
[
|
|
153
|
+
# First: the tuning migrations below build table names from
|
|
154
|
+
# pgmq.meta, which a mixed-case row spells unlike its table.
|
|
155
|
+
*queue_name_case_migrations,
|
|
146
156
|
*uniqueness_key_migrations,
|
|
147
157
|
*job_stats_migrations,
|
|
148
158
|
*stream_stats_migrations,
|
|
@@ -328,6 +338,23 @@ module Pgbus
|
|
|
328
338
|
[:tune_autovacuum]
|
|
329
339
|
end
|
|
330
340
|
|
|
341
|
+
# Issue #633: a queue an older pgbus created in mixed case shares its
|
|
342
|
+
# lower-case table with the spelling pgbus now uses. An unreadable
|
|
343
|
+
# pgmq.meta queues nothing: the doctor's "Queue name case" check says why.
|
|
344
|
+
def queue_name_case_migrations
|
|
345
|
+
return [] unless pgmq_schema_exists?
|
|
346
|
+
return [] unless mixed_case_queue?
|
|
347
|
+
|
|
348
|
+
[:fold_queue_name_case]
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
def mixed_case_queue?
|
|
352
|
+
prefix = "#{Pgbus.configuration.queue_prefix.to_s.downcase}_"
|
|
353
|
+
connection.select_value(MIXED_CASE_QUEUE_SQL.gsub("$1", connection.quote(prefix))).present?
|
|
354
|
+
rescue StandardError
|
|
355
|
+
false
|
|
356
|
+
end
|
|
357
|
+
|
|
331
358
|
# Fillfactor tuning: check if any PGMQ queue table already has
|
|
332
359
|
# fillfactor applied. If not, queue the migration.
|
|
333
360
|
def fillfactor_migrations
|
|
@@ -16,7 +16,22 @@ module Pgbus
|
|
|
16
16
|
# pgbus.job_completed — job archived successfully
|
|
17
17
|
# pgbus.job_failed — job raised; carries :exception_object
|
|
18
18
|
# pgbus.job_dead_lettered — job exceeded max_retries and was DLQ-routed;
|
|
19
|
-
# :error is the last recorded error class (or nil)
|
|
19
|
+
# :error is the last recorded error class (or nil);
|
|
20
|
+
# :reason is max_retries_exceeded, or uniqueness_conflict
|
|
21
|
+
# for a :while_executing copy with no failed attempt whose
|
|
22
|
+
# key's row is not its own (skipped while another message
|
|
23
|
+
# held the key); also uniqueness_conflict when that row
|
|
24
|
+
# lookup itself fails
|
|
25
|
+
# pgbus.job_skipped — a :while_executing copy was read while another
|
|
26
|
+
# message held its key and left for a later delivery
|
|
27
|
+
# (on_conflict :reject or :log)
|
|
28
|
+
# payload: queue, job_class, job_id, provider_job_id,
|
|
29
|
+
# msg_id, read_ct, on_conflict, vt (seconds until it
|
|
30
|
+
# is delivered again)
|
|
31
|
+
# pgbus.job_discarded — a copy was completed without running
|
|
32
|
+
# payload: queue, job_class, job_id, provider_job_id,
|
|
33
|
+
# msg_id, reason ("uniqueness": a :while_executing copy
|
|
34
|
+
# under on_conflict: :discard)
|
|
20
35
|
# pgbus.job_visibility_extended — heartbeat re-armed a running job's visibility timeout
|
|
21
36
|
# payload: queue, job_class, msg_id, vt, extensions
|
|
22
37
|
# pgbus.event_processed — event handler succeeded
|
|
@@ -60,6 +60,8 @@ module Pgbus
|
|
|
60
60
|
subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
|
|
61
61
|
subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
|
|
62
62
|
subscribe("pgbus.job_dead_lettered") { |event| on_job_dead_lettered(event) },
|
|
63
|
+
subscribe("pgbus.job_skipped") { |event| on_job_skipped(event) },
|
|
64
|
+
subscribe("pgbus.job_discarded") { |event| on_job_discarded(event) },
|
|
63
65
|
subscribe("pgbus.job_visibility_extended") { |event| on_job_visibility_extended(event) },
|
|
64
66
|
subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
|
|
65
67
|
subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
|
|
@@ -121,11 +123,35 @@ module Pgbus
|
|
|
121
123
|
)
|
|
122
124
|
end
|
|
123
125
|
|
|
126
|
+
# reason: max_retries_exceeded, or uniqueness_conflict for a
|
|
127
|
+
# :while_executing duplicate skipped until it ran out of reads (#629).
|
|
124
128
|
def on_job_dead_lettered(event)
|
|
125
129
|
payload = event.payload
|
|
126
130
|
backend.increment(
|
|
127
131
|
"#{METRIC_PREFIX}queue_job_count", 1,
|
|
128
|
-
compact(queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered"
|
|
132
|
+
compact(queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered",
|
|
133
|
+
reason: payload[:reason])
|
|
134
|
+
)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# A :while_executing duplicate left for a later delivery because
|
|
138
|
+
# another message held its key (issue #629): counted per skip, so one
|
|
139
|
+
# copy waiting out a long holder counts several times.
|
|
140
|
+
def on_job_skipped(event)
|
|
141
|
+
payload = event.payload
|
|
142
|
+
backend.increment(
|
|
143
|
+
"#{METRIC_PREFIX}queue_job_count", 1,
|
|
144
|
+
compact(queue: payload[:queue], job_class: payload[:job_class], status: "skipped", reason: "uniqueness")
|
|
145
|
+
)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# A :while_executing duplicate dropped under on_conflict: :discard.
|
|
149
|
+
def on_job_discarded(event)
|
|
150
|
+
payload = event.payload
|
|
151
|
+
backend.increment(
|
|
152
|
+
"#{METRIC_PREFIX}queue_job_count", 1,
|
|
153
|
+
compact(queue: payload[:queue], job_class: payload[:job_class], status: "discarded",
|
|
154
|
+
reason: payload[:reason])
|
|
129
155
|
)
|
|
130
156
|
end
|
|
131
157
|
|
|
@@ -94,7 +94,8 @@ module Pgbus
|
|
|
94
94
|
return waiter.value if waiter
|
|
95
95
|
|
|
96
96
|
# Stopped: complete on the caller's thread, with the same results.
|
|
97
|
-
@client.public_send(@batch_op, queue, [msg_id], prefixed: false
|
|
97
|
+
done = @client.public_send(@batch_op, queue, [msg_id], prefixed: false, job_completion: true)
|
|
98
|
+
done.empty? ? :already_archived : :archived
|
|
98
99
|
end
|
|
99
100
|
|
|
100
101
|
# Flushes whatever is pending, then joins the flusher. Past timeout the
|
|
@@ -157,7 +158,8 @@ module Pgbus
|
|
|
157
158
|
# is absent. A duplicated id (the same message held by two threads after
|
|
158
159
|
# a redelivery) is archived once, so only its first waiter gets :archived.
|
|
159
160
|
def flush_slice(queue, slice)
|
|
160
|
-
archived = @client.public_send(@batch_op, queue, slice.map(&:first).uniq, prefixed: false)
|
|
161
|
+
archived = @client.public_send(@batch_op, queue, slice.map(&:first).uniq, prefixed: false, job_completion: true)
|
|
162
|
+
.to_set(&:to_i)
|
|
161
163
|
slice.each { |msg_id, waiter| waiter.resolve(archived.delete?(msg_id) ? :archived : :already_archived) }
|
|
162
164
|
rescue Exception => e # rubocop:disable Lint/RescueException -- delivered to the waiters, never swallowed
|
|
163
165
|
log_flush_failure(queue, slice, e) unless e.is_a?(StandardError)
|
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -194,6 +194,7 @@ module Pgbus
|
|
|
194
194
|
POOL_TERMINATION_WAIT = 5
|
|
195
195
|
|
|
196
196
|
def run
|
|
197
|
+
quiet_active_job_logs
|
|
197
198
|
setup_signals
|
|
198
199
|
start_heartbeat
|
|
199
200
|
resolve_wildcard_queues
|
|
@@ -282,6 +283,18 @@ module Pgbus
|
|
|
282
283
|
|
|
283
284
|
private
|
|
284
285
|
|
|
286
|
+
# config.quiet_active_job_logs (issue #618): only worker processes perform
|
|
287
|
+
# jobs, so only they drop the per-job "Performing …" / "Performed …"
|
|
288
|
+
# lines. ActiveJob::LogSubscriber attaches when active_job/base loads,
|
|
289
|
+
# which an app with no job class to eager-load has not done yet, so load
|
|
290
|
+
# it first (defined? does not trigger ActiveJob's autoload of Base).
|
|
291
|
+
def quiet_active_job_logs
|
|
292
|
+
return unless config.quiet_active_job_logs && defined?(::ActiveJob::Base)
|
|
293
|
+
|
|
294
|
+
require "active_job/base"
|
|
295
|
+
Pgbus::ActiveJob::QuietLogs.apply!
|
|
296
|
+
end
|
|
297
|
+
|
|
285
298
|
# One loop step (issue #486): feed buffered claims to free slots, read
|
|
286
299
|
# the deficit (free slots + read_ahead - buffered, capped by
|
|
287
300
|
# prefetch_limit), feed again. With read_ahead 0 the buffer never
|
|
@@ -510,11 +523,18 @@ module Pgbus
|
|
|
510
523
|
def process_message(message, queue_name, source_queue: nil)
|
|
511
524
|
result = @executor.execute(message, queue_name, source_queue: source_queue)
|
|
512
525
|
@jobs_processed.increment
|
|
513
|
-
|
|
514
|
-
|
|
526
|
+
# A skipped uniqueness duplicate comes back later: counting it now
|
|
527
|
+
# would count one job twice (issue #629).
|
|
528
|
+
@rate_counter.increment(:processed) unless result == :skipped
|
|
529
|
+
case result
|
|
530
|
+
when :failed
|
|
515
531
|
@jobs_failed.increment
|
|
516
532
|
@rate_counter.increment(:failed)
|
|
517
533
|
@circuit_breaker.record_failure(queue_name)
|
|
534
|
+
when :skipped, :discarded
|
|
535
|
+
# No job ran: neither a failure nor a success that would wipe the
|
|
536
|
+
# failure streak of a queue whose real jobs keep failing.
|
|
537
|
+
nil
|
|
518
538
|
else
|
|
519
539
|
@circuit_breaker.record_success(queue_name)
|
|
520
540
|
end
|
|
@@ -569,7 +589,12 @@ module Pgbus
|
|
|
569
589
|
prefix = "#{config.queue_prefix}_"
|
|
570
590
|
if (match = MISSING_QUEUE_REGEX.match(error.message))
|
|
571
591
|
physical_name = match[1]
|
|
572
|
-
|
|
592
|
+
# Postgres names the table as PGMQ spells it (lower case, issue #633;
|
|
593
|
+
# separators normalized), so find the queue as configured.
|
|
594
|
+
# A priority sub-queue (_pN) belongs to its base queue.
|
|
595
|
+
logical_name = @queues.find { |q| config.priority_queue_names(q).include?(physical_name) } ||
|
|
596
|
+
@queues.find { |q| config.queue_name(q) == physical_name } ||
|
|
597
|
+
physical_name.delete_prefix(prefix)
|
|
573
598
|
if @queues.delete(logical_name)
|
|
574
599
|
Pgbus.logger.warn { "[Pgbus] Evicted deleted queue '#{logical_name}' (#{physical_name}) from worker" }
|
|
575
600
|
end
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
# Issue #633 upgrade path. PGMQ lower-cases its table names
|
|
5
|
+
# (pgmq.format_table_name), so a queue an older pgbus created as
|
|
6
|
+
# "pgbus_Orders" lives in pgmq.q_pgbus_orders while pgmq.meta records
|
|
7
|
+
# "pgbus_Orders". pgbus now spells every queue in lower case; this renames
|
|
8
|
+
# the old rows to match their tables.
|
|
9
|
+
#
|
|
10
|
+
# Only rows change. The queue tables are shared by both spellings, so nothing
|
|
11
|
+
# here drops, renames or copies a table: dropping either spelling's queue
|
|
12
|
+
# through PGMQ would remove the shared table.
|
|
13
|
+
#
|
|
14
|
+
# Run by the pgbus:fold_queue_name_case migration and by
|
|
15
|
+
# `rake pgbus:queues:fold_case` (for a row an older process recreated after
|
|
16
|
+
# the migration ran). Idempotent. Takes an ActiveRecord connection and
|
|
17
|
+
# folds pgbus's queues only (those named "<prefix>_…" in any case), so
|
|
18
|
+
# another PGMQ user of the same schema keeps its spelling and its NOTIFY
|
|
19
|
+
# channel.
|
|
20
|
+
module QueueNameFold
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
def run!(connection, prefix: Pgbus.configuration.queue_prefix)
|
|
24
|
+
fold_pgmq_meta(connection, "#{prefix.to_s.downcase}_")
|
|
25
|
+
fold_unique_column(connection, :pgbus_stream_queues, prefer: "")
|
|
26
|
+
fold_unique_column(connection, :pgbus_queue_states, prefer: "paused DESC,")
|
|
27
|
+
fold_column(connection, :pgbus_uniqueness_keys, :queue_name)
|
|
28
|
+
fold_failed_event_source_queues(connection)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# One mixed-case pgmq.meta row at a time: add the lower-case row, move its
|
|
32
|
+
# topic bindings, delete the old row (and its throttle and binding rows),
|
|
33
|
+
# then re-arm the insert-NOTIFY trigger under the new name. PGMQ 1.13.1's
|
|
34
|
+
# trigger NOTIFYs on the channel of the name it was armed with, which
|
|
35
|
+
# listeners now spell in lower case. Where both spellings exist, the
|
|
36
|
+
# lower-case row's throttle wins.
|
|
37
|
+
def fold_pgmq_meta(connection, own_prefix)
|
|
38
|
+
prefix = connection.quote(own_prefix)
|
|
39
|
+
connection.execute(<<~SQL)
|
|
40
|
+
DO $$
|
|
41
|
+
DECLARE
|
|
42
|
+
q RECORD;
|
|
43
|
+
folded TEXT;
|
|
44
|
+
throttle INTEGER;
|
|
45
|
+
has_throttle BOOLEAN := to_regclass('pgmq.notify_insert_throttle') IS NOT NULL;
|
|
46
|
+
has_bindings BOOLEAN := to_regclass('pgmq.topic_bindings') IS NOT NULL;
|
|
47
|
+
BEGIN
|
|
48
|
+
FOR q IN
|
|
49
|
+
SELECT queue_name, is_partitioned, is_unlogged, created_at
|
|
50
|
+
FROM pgmq.meta
|
|
51
|
+
WHERE queue_name <> lower(queue_name)
|
|
52
|
+
AND left(lower(queue_name), length(#{prefix})) = #{prefix}
|
|
53
|
+
ORDER BY queue_name
|
|
54
|
+
LOOP
|
|
55
|
+
folded := lower(q.queue_name);
|
|
56
|
+
|
|
57
|
+
INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged, created_at)
|
|
58
|
+
VALUES (folded, q.is_partitioned, q.is_unlogged, q.created_at)
|
|
59
|
+
ON CONFLICT DO NOTHING;
|
|
60
|
+
|
|
61
|
+
throttle := NULL;
|
|
62
|
+
IF has_throttle THEN
|
|
63
|
+
SELECT COALESCE(
|
|
64
|
+
(SELECT throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = folded),
|
|
65
|
+
(SELECT throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = q.queue_name)
|
|
66
|
+
) INTO throttle;
|
|
67
|
+
DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = q.queue_name;
|
|
68
|
+
END IF;
|
|
69
|
+
|
|
70
|
+
IF has_bindings THEN
|
|
71
|
+
INSERT INTO pgmq.topic_bindings (pattern, queue_name, bound_at)
|
|
72
|
+
SELECT pattern, folded, bound_at FROM pgmq.topic_bindings WHERE queue_name = q.queue_name
|
|
73
|
+
ON CONFLICT DO NOTHING;
|
|
74
|
+
DELETE FROM pgmq.topic_bindings WHERE queue_name = q.queue_name;
|
|
75
|
+
END IF;
|
|
76
|
+
|
|
77
|
+
DELETE FROM pgmq.meta WHERE queue_name = q.queue_name;
|
|
78
|
+
|
|
79
|
+
IF throttle IS NOT NULL THEN
|
|
80
|
+
PERFORM pgmq.enable_notify_insert(folded, throttle);
|
|
81
|
+
END IF;
|
|
82
|
+
END LOOP;
|
|
83
|
+
END $$;
|
|
84
|
+
SQL
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# A column with a unique index: keep one row per lower-case name (the
|
|
88
|
+
# `prefer` order first, then the row already in lower case, then the
|
|
89
|
+
# oldest), delete the rest, then lower-case what is left.
|
|
90
|
+
def fold_unique_column(connection, table, prefer:)
|
|
91
|
+
return unless connection.table_exists?(table)
|
|
92
|
+
|
|
93
|
+
connection.execute(<<~SQL)
|
|
94
|
+
DELETE FROM #{table} WHERE id IN (
|
|
95
|
+
SELECT id FROM (
|
|
96
|
+
SELECT id, row_number() OVER (
|
|
97
|
+
PARTITION BY lower(queue_name)
|
|
98
|
+
ORDER BY #{prefer} (queue_name = lower(queue_name)) DESC, id
|
|
99
|
+
) AS rank
|
|
100
|
+
FROM #{table}
|
|
101
|
+
WHERE lower(queue_name) IN (SELECT lower(queue_name) FROM #{table} WHERE queue_name <> lower(queue_name))
|
|
102
|
+
) ranked
|
|
103
|
+
WHERE rank > 1
|
|
104
|
+
)
|
|
105
|
+
SQL
|
|
106
|
+
fold_column(connection, table, :queue_name)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def fold_column(connection, table, column)
|
|
110
|
+
return unless connection.table_exists?(table) && connection.column_exists?(table, column)
|
|
111
|
+
|
|
112
|
+
connection.execute("UPDATE #{table} SET #{column} = lower(#{column}) WHERE #{column} <> lower(#{column})")
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# source_queue is the physical queue a failure was read from, now spelled in
|
|
116
|
+
# lower case. queue_name is the executor's logical name, as configured, and
|
|
117
|
+
# stays. A message that failed under both spellings (before and after the
|
|
118
|
+
# deploy) has two rows: one per (queue_name, msg_id, folded source) stays,
|
|
119
|
+
# the lower-case one if there is one (written after the deploy), else the
|
|
120
|
+
# newest, so the (queue_name, msg_id, source_queue) unique index holds.
|
|
121
|
+
def fold_failed_event_source_queues(connection)
|
|
122
|
+
return unless connection.table_exists?(:pgbus_failed_events) && connection.column_exists?(:pgbus_failed_events, :source_queue)
|
|
123
|
+
|
|
124
|
+
connection.execute(<<~SQL)
|
|
125
|
+
DELETE FROM pgbus_failed_events WHERE id IN (
|
|
126
|
+
SELECT id FROM (
|
|
127
|
+
SELECT id, row_number() OVER (
|
|
128
|
+
PARTITION BY queue_name, msg_id, lower(source_queue)
|
|
129
|
+
ORDER BY (source_queue = lower(source_queue)) DESC, id DESC
|
|
130
|
+
) AS rank
|
|
131
|
+
FROM pgbus_failed_events
|
|
132
|
+
WHERE (queue_name, msg_id, lower(source_queue)) IN (
|
|
133
|
+
SELECT queue_name, msg_id, lower(source_queue) FROM pgbus_failed_events
|
|
134
|
+
WHERE source_queue <> lower(source_queue)
|
|
135
|
+
)
|
|
136
|
+
) ranked
|
|
137
|
+
WHERE rank > 1
|
|
138
|
+
)
|
|
139
|
+
SQL
|
|
140
|
+
fold_column(connection, :pgbus_failed_events, :source_queue)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
private_class_method :fold_pgmq_meta, :fold_unique_column, :fold_column, :fold_failed_event_source_queues
|
|
144
|
+
end
|
|
145
|
+
end
|
|
@@ -9,7 +9,9 @@ module Pgbus
|
|
|
9
9
|
module QueueNameValidator
|
|
10
10
|
# PostgreSQL's NAMEDATALEN caps identifiers at 63 bytes, but the
|
|
11
11
|
# effective limit is tighter: pgmq-ruby (our transport gem) rejects
|
|
12
|
-
# any queue name with `length >=
|
|
12
|
+
# any queue name with `length >= PGMQ::QueueName::MAX_LENGTH` (48) in
|
|
13
|
+
# `PGMQ::QueueName.validate!`, whose error has said "maximum length of
|
|
14
|
+
# 47 characters" since pgmq-ruby 0.8.2 (48 before).
|
|
13
15
|
# That leaves an actual usable ceiling of 47 characters for the
|
|
14
16
|
# fully-prefixed name (`<queue_prefix>_<logical_name>`), which is what
|
|
15
17
|
# this constant expresses. pgmq-ruby picked 48 to leave headroom for
|
|
@@ -26,6 +28,10 @@ module Pgbus
|
|
|
26
28
|
# Only alphanumeric characters and underscores are allowed.
|
|
27
29
|
VALID_QUEUE_NAME_PATTERN = /\A[a-zA-Z0-9_]+\z/
|
|
28
30
|
|
|
31
|
+
# A valid name already in lower case: what normalize returns.
|
|
32
|
+
CANONICAL_QUEUE_NAME = /\A[a-z0-9_]+\z/
|
|
33
|
+
private_constant :CANONICAL_QUEUE_NAME
|
|
34
|
+
|
|
29
35
|
module_function
|
|
30
36
|
|
|
31
37
|
# Validates a queue name for safe SQL identifier use.
|
|
@@ -58,14 +64,26 @@ module Pgbus
|
|
|
58
64
|
# (`Pgbus.stream([user, :notifications])` → `"user_gid:notifications"`),
|
|
59
65
|
# so they must map to a safe character rather than be stripped —
|
|
60
66
|
# otherwise `"a:b"` and `"ab"` would collide on the same queue.
|
|
67
|
+
#
|
|
68
|
+
# The result is lower case (issue #633): PGMQ lower-cases its table
|
|
69
|
+
# names (`pgmq.format_table_name`), so "Orders" and "orders" share
|
|
70
|
+
# `pgmq.q_orders` while `pgmq.meta` records each spelling as its own
|
|
71
|
+
# queue. Spelling every name the way its table is gives one name per
|
|
72
|
+
# table. `validate!` and `sanitize!` keep case: they also run on names
|
|
73
|
+
# read back from `pgmq.meta`, which an install from before #633 may
|
|
74
|
+
# still hold in mixed case (`rails generate pgbus:fold_queue_name_case`).
|
|
61
75
|
def normalize(name)
|
|
62
76
|
name = name.to_s
|
|
63
|
-
|
|
77
|
+
# Every enqueue and read names its queue through here: a name already
|
|
78
|
+
# spelled as its table costs one scan and no allocation.
|
|
79
|
+
return name if name.length <= MAX_QUEUE_NAME_LENGTH && CANONICAL_QUEUE_NAME.match?(name)
|
|
80
|
+
return validate!(name).downcase if VALID_QUEUE_NAME_PATTERN.match?(name)
|
|
64
81
|
|
|
65
82
|
normalized = name.gsub(/[-.:]/, "_") # hyphens/dots/colons → underscores
|
|
66
83
|
.gsub(/[^a-zA-Z0-9_]/, "") # strip remaining invalid chars
|
|
67
84
|
.gsub(/_+/, "_") # collapse consecutive underscores
|
|
68
85
|
.gsub(/\A_|_\z/, "") # strip leading/trailing underscores
|
|
86
|
+
normalized.downcase!
|
|
69
87
|
validate!(normalized)
|
|
70
88
|
normalized
|
|
71
89
|
end
|