pgbus 0.7.3 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -15
- data/README.md +236 -0
- data/app/controllers/pgbus/events_controller.rb +98 -2
- data/app/views/pgbus/events/_pending_table.html.erb +148 -0
- data/app/views/pgbus/events/index.html.erb +21 -1
- data/config/locales/da.yml +109 -0
- data/config/locales/de.yml +109 -0
- data/config/locales/en.yml +47 -0
- data/config/locales/es.yml +109 -0
- data/config/locales/fi.yml +109 -0
- data/config/locales/fr.yml +109 -0
- data/config/locales/it.yml +109 -0
- data/config/locales/ja.yml +109 -0
- data/config/locales/nb.yml +109 -0
- data/config/locales/nl.yml +109 -0
- data/config/locales/pt.yml +109 -0
- data/config/locales/sv.yml +109 -0
- data/config/routes.rb +7 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +5 -0
- data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +30 -0
- data/lib/generators/pgbus/tune_fillfactor_generator.rb +51 -0
- data/lib/pgbus/client.rb +61 -9
- data/lib/pgbus/configuration.rb +8 -0
- data/lib/pgbus/engine.rb +16 -4
- data/lib/pgbus/event_bus/publisher.rb +5 -3
- data/lib/pgbus/generators/migration_detector.rb +31 -3
- data/lib/pgbus/process/dispatcher.rb +16 -0
- data/lib/pgbus/process/supervisor.rb +11 -1
- data/lib/pgbus/recurring/config_loader.rb +24 -0
- data/lib/pgbus/table_maintenance.rb +110 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +147 -2
- metadata +5 -1
data/config/locales/pt.yml
CHANGED
|
@@ -66,6 +66,12 @@ pt:
|
|
|
66
66
|
index:
|
|
67
67
|
discard_all: Descartar Tudo
|
|
68
68
|
discard_all_confirm: Descartar permanentemente todas as mensagens DLQ?
|
|
69
|
+
discard_selected: Descartar Selecionados
|
|
70
|
+
discard_selected_confirm: Descartar mensagens DLQ selecionadas?
|
|
71
|
+
discarded_selected:
|
|
72
|
+
one: 1 mensagem DLQ descartada.
|
|
73
|
+
other: "%{count} mensagens DLQ descartadas."
|
|
74
|
+
none_selected: Nenhuma mensagem selecionada.
|
|
69
75
|
retry_all: Tentar Novamente Tudo
|
|
70
76
|
retry_all_confirm: Tentar novamente todas as mensagens DLQ?
|
|
71
77
|
title: Fila de Mensagens Mortas
|
|
@@ -113,7 +119,34 @@ pt:
|
|
|
113
119
|
delete_title: Excluir
|
|
114
120
|
ok: OK
|
|
115
121
|
events:
|
|
122
|
+
flash:
|
|
123
|
+
discard_failed: Não foi possível descartar o evento.
|
|
124
|
+
discarded: Evento descartado.
|
|
125
|
+
discarded_selected:
|
|
126
|
+
one: 1 evento descartado.
|
|
127
|
+
other: "%{count} eventos descartados."
|
|
128
|
+
mark_handled_failed: Não foi possível marcar o evento como tratado.
|
|
129
|
+
marked_handled: Evento marcado como tratado.
|
|
130
|
+
none_selected: Nenhum evento selecionado.
|
|
131
|
+
payload_update_failed: Não foi possível atualizar o payload. Verifique se o JSON é válido.
|
|
132
|
+
payload_updated: Payload atualizado e evento reenfileirado.
|
|
133
|
+
replay_failed: Evento não encontrado ou não pôde ser reproduzido.
|
|
134
|
+
replayed: Evento reproduzido.
|
|
135
|
+
reroute_failed: Não foi possível redirecionar o evento.
|
|
136
|
+
rerouted: Evento redirecionado para o manipulador alvo.
|
|
116
137
|
index:
|
|
138
|
+
discard_all: Descartar Todos
|
|
139
|
+
discard_all_confirm: Descartar todos os eventos pendentes? Esta ação não pode ser desfeita.
|
|
140
|
+
discard_selected: Descartar Selecionados
|
|
141
|
+
discard_selected_confirm: Descartar eventos selecionados?
|
|
142
|
+
pending_empty: Nenhum evento pendente
|
|
143
|
+
pending_headers:
|
|
144
|
+
enqueued: Enfileirado
|
|
145
|
+
handler_queue: Fila do Manipulador
|
|
146
|
+
id: ID
|
|
147
|
+
reads: Leituras
|
|
148
|
+
routing_key: Chave de Roteamento
|
|
149
|
+
pending_title: Eventos Pendentes
|
|
117
150
|
processed_empty: Nenhum evento processado ainda
|
|
118
151
|
processed_headers:
|
|
119
152
|
event_id: ID do evento
|
|
@@ -127,6 +160,26 @@ pt:
|
|
|
127
160
|
queue: Fila
|
|
128
161
|
subscribers_title: Assinantes registrados
|
|
129
162
|
title: Eventos
|
|
163
|
+
pending_table:
|
|
164
|
+
arguments: Payload
|
|
165
|
+
discard: Descartar
|
|
166
|
+
discard_confirm: Descartar este evento? Ele será arquivado.
|
|
167
|
+
edit_payload: Editar e Tentar Novamente
|
|
168
|
+
edit_payload_confirm: Atualizar payload e reenfileirar este evento?
|
|
169
|
+
edit_payload_label: 'Payload JSON:'
|
|
170
|
+
event_id: 'ID do Evento:'
|
|
171
|
+
full_json_payload: Payload JSON Completo
|
|
172
|
+
headers_section: Cabeçalhos
|
|
173
|
+
mark_handled: Marcar como Tratado
|
|
174
|
+
mark_handled_confirm: Marcar este evento como tratado? Ele será arquivado e ignorado na reprodução.
|
|
175
|
+
metadata: Metadados
|
|
176
|
+
metadata_labels:
|
|
177
|
+
last_read: 'Última leitura:'
|
|
178
|
+
read_count: 'Contagem de leituras:'
|
|
179
|
+
visible_at: 'Visível em:'
|
|
180
|
+
reroute: Redirecionar
|
|
181
|
+
reroute_confirm: Redirecionar este evento para um manipulador diferente?
|
|
182
|
+
reroute_label: 'Manipulador alvo:'
|
|
130
183
|
show:
|
|
131
184
|
back: Voltar para eventos
|
|
132
185
|
labels:
|
|
@@ -136,6 +189,9 @@ pt:
|
|
|
136
189
|
not_found: Evento não encontrado
|
|
137
190
|
title: Evento %{event_id}
|
|
138
191
|
helpers:
|
|
192
|
+
bulk_select_all: Selecionar todos
|
|
193
|
+
bulk_select_row: Selecionar %{id}
|
|
194
|
+
bulk_selected: selecionado
|
|
139
195
|
paused_badge: Pausado
|
|
140
196
|
queue_badge:
|
|
141
197
|
dlq: DLQ
|
|
@@ -150,11 +206,22 @@ pt:
|
|
|
150
206
|
show:
|
|
151
207
|
charts:
|
|
152
208
|
failed_to_load: Falha ao carregar dados do gráfico
|
|
209
|
+
latency: Latência da Fila (ms)
|
|
210
|
+
latency_avg: Média
|
|
211
|
+
latency_p95: P95
|
|
153
212
|
no_data: Sem dados ainda
|
|
154
213
|
series_name: Tarefas/min
|
|
155
214
|
status_distribution: Distribuição de status
|
|
156
215
|
throughput: Taxa de transferência (tarefas/min)
|
|
157
216
|
description_html: Métricas de desempenho de trabalho para os últimos %{range}
|
|
217
|
+
latency_by_queue:
|
|
218
|
+
empty: Ainda não há dados de latência
|
|
219
|
+
headers:
|
|
220
|
+
avg: Média (ms)
|
|
221
|
+
count: Contagem
|
|
222
|
+
p95: P95 (ms)
|
|
223
|
+
queue: Fila
|
|
224
|
+
title: Latência por Fila
|
|
158
225
|
slowest:
|
|
159
226
|
empty: Nenhuma estatística de tarefa ainda
|
|
160
227
|
headers:
|
|
@@ -163,11 +230,32 @@ pt:
|
|
|
163
230
|
job_class: Classe da tarefa
|
|
164
231
|
max: Máximo
|
|
165
232
|
title: Classes de Trabalho Mais Lentas (duração média)
|
|
233
|
+
streams:
|
|
234
|
+
summary:
|
|
235
|
+
active: Ativo
|
|
236
|
+
avg_fanout: Média de Ramificações
|
|
237
|
+
broadcasts: Transmissões
|
|
238
|
+
connects: Conexões
|
|
239
|
+
disconnects: Desconexões
|
|
240
|
+
title: Streams em Tempo Real
|
|
241
|
+
top:
|
|
242
|
+
empty: Nenhuma atividade de stream registrada na janela selecionada
|
|
243
|
+
headers:
|
|
244
|
+
avg_fanout: Média de Ramificações
|
|
245
|
+
avg_ms: Média de Despacho
|
|
246
|
+
broadcasts: Transmissões
|
|
247
|
+
stream: Stream
|
|
248
|
+
title: Principais Streams por Volume de Transmissão
|
|
166
249
|
summary:
|
|
167
250
|
avg_duration: Duração Média
|
|
251
|
+
avg_latency: Latência Média
|
|
252
|
+
avg_retries: Média de Tentativas
|
|
168
253
|
dead_lettered: Cartas Mortas
|
|
169
254
|
failed: Falhou
|
|
170
255
|
max_duration: Duração Máxima
|
|
256
|
+
p50_latency: Latência P50
|
|
257
|
+
p95_latency: Latência P95
|
|
258
|
+
p99_latency: Latência P99
|
|
171
259
|
succeeded: Bem-sucedido
|
|
172
260
|
total_jobs: Total de Trabalhos
|
|
173
261
|
time_ranges:
|
|
@@ -222,6 +310,12 @@ pt:
|
|
|
222
310
|
discard_all: Descartar Todos
|
|
223
311
|
discard_all_confirm: Descartar todos os trabalhos falhados?
|
|
224
312
|
discard_all_enqueued_notice: Descartados %{count} trabalhos na fila e bloqueios liberados.
|
|
313
|
+
discard_selected: Descartar Selecionados
|
|
314
|
+
discard_selected_confirm: Descartar itens selecionados?
|
|
315
|
+
discarded_selected:
|
|
316
|
+
one: 1 item selecionado descartado.
|
|
317
|
+
other: "%{count} itens selecionados descartados."
|
|
318
|
+
none_selected: Nenhum item selecionado.
|
|
225
319
|
retry_all: Tentar Novamente Todos
|
|
226
320
|
retry_all_confirm: Tentar novamente todos os trabalhos falhados?
|
|
227
321
|
title: Trabalhos
|
|
@@ -259,13 +353,28 @@ pt:
|
|
|
259
353
|
toggle_menu: Alternar menu
|
|
260
354
|
locks:
|
|
261
355
|
index:
|
|
356
|
+
all_locks_discarded:
|
|
357
|
+
one: 1 bloqueio descartado.
|
|
358
|
+
other: "%{count} bloqueios descartados."
|
|
262
359
|
description: Bloqueios de exclusividade ativos impedindo a execução duplicada do trabalho
|
|
360
|
+
discard: Descartar
|
|
361
|
+
discard_all: Descartar Todos
|
|
362
|
+
discard_all_confirm: Descartar permanentemente todos os bloqueios? Isso pode permitir a execução duplicada de trabalhos.
|
|
363
|
+
discard_confirm: Descartar este bloqueio? O trabalho associado pode ser enfileirado novamente.
|
|
364
|
+
discard_selected: Descartar Selecionados
|
|
365
|
+
discard_selected_confirm: Descartar bloqueios selecionados?
|
|
263
366
|
empty: Nenhum bloqueio ativo
|
|
264
367
|
headers:
|
|
265
368
|
age: Idade
|
|
266
369
|
lock_key: Chave de Bloqueio
|
|
267
370
|
msg_id: ID da mensagem
|
|
268
371
|
queue_name: Fila
|
|
372
|
+
lock_discard_failed: Não foi possível descartar o bloqueio.
|
|
373
|
+
lock_discarded: Bloqueio descartado.
|
|
374
|
+
locks_discarded:
|
|
375
|
+
one: 1 bloqueio descartado.
|
|
376
|
+
other: "%{count} bloqueios descartados."
|
|
377
|
+
none_selected: Nenhum bloqueio selecionado.
|
|
269
378
|
title: Chaves de unicidade
|
|
270
379
|
outbox:
|
|
271
380
|
index:
|
data/config/locales/sv.yml
CHANGED
|
@@ -66,6 +66,12 @@ sv:
|
|
|
66
66
|
index:
|
|
67
67
|
discard_all: Kassera alla
|
|
68
68
|
discard_all_confirm: Kassera permanent alla DLQ-meddelanden?
|
|
69
|
+
discard_selected: Kassera valda
|
|
70
|
+
discard_selected_confirm: Kassera valda DLQ-meddelanden?
|
|
71
|
+
discarded_selected:
|
|
72
|
+
one: Kasserade 1 DLQ-meddelande.
|
|
73
|
+
other: Kasserade %{count} DLQ-meddelanden.
|
|
74
|
+
none_selected: Inga meddelanden valda.
|
|
69
75
|
retry_all: Försök igen alla
|
|
70
76
|
retry_all_confirm: Försök igen alla DLQ-meddelanden?
|
|
71
77
|
title: Dead Letter-kö
|
|
@@ -113,7 +119,34 @@ sv:
|
|
|
113
119
|
delete_title: Ta bort
|
|
114
120
|
ok: OK
|
|
115
121
|
events:
|
|
122
|
+
flash:
|
|
123
|
+
discard_failed: Kunde inte kassera händelsen.
|
|
124
|
+
discarded: Händelse kasserad.
|
|
125
|
+
discarded_selected:
|
|
126
|
+
one: Kasserade 1 händelse.
|
|
127
|
+
other: Kasserade %{count} händelser.
|
|
128
|
+
mark_handled_failed: Kunde inte markera händelsen som hanterad.
|
|
129
|
+
marked_handled: Händelse markerad som hanterad.
|
|
130
|
+
none_selected: Inga händelser valda.
|
|
131
|
+
payload_update_failed: Kunde inte uppdatera payload. Kontrollera att JSON är giltig.
|
|
132
|
+
payload_updated: Payload uppdaterad och händelsen återköad.
|
|
133
|
+
replay_failed: Händelsen hittades inte eller kunde inte spelas upp.
|
|
134
|
+
replayed: Händelsen spelades upp.
|
|
135
|
+
reroute_failed: Kunde inte omdirigera händelsen.
|
|
136
|
+
rerouted: Händelsen omdirigerades till målhanteraren.
|
|
116
137
|
index:
|
|
138
|
+
discard_all: Kassera alla
|
|
139
|
+
discard_all_confirm: Kassera alla väntande händelser? Detta kan inte ångras.
|
|
140
|
+
discard_selected: Kassera valda
|
|
141
|
+
discard_selected_confirm: Kassera valda händelser?
|
|
142
|
+
pending_empty: Inga väntande händelser
|
|
143
|
+
pending_headers:
|
|
144
|
+
enqueued: Köad
|
|
145
|
+
handler_queue: Handler-kö
|
|
146
|
+
id: ID
|
|
147
|
+
reads: Läsningar
|
|
148
|
+
routing_key: Routing-nyckel
|
|
149
|
+
pending_title: Väntande händelser
|
|
117
150
|
processed_empty: Inga händelser behandlade än
|
|
118
151
|
processed_headers:
|
|
119
152
|
event_id: Händelse-ID
|
|
@@ -127,6 +160,26 @@ sv:
|
|
|
127
160
|
queue: Kö
|
|
128
161
|
subscribers_title: Registrerade prenumeranter
|
|
129
162
|
title: Händelser
|
|
163
|
+
pending_table:
|
|
164
|
+
arguments: Payload
|
|
165
|
+
discard: Kassera
|
|
166
|
+
discard_confirm: Kassera denna händelse? Den kommer att arkiveras.
|
|
167
|
+
edit_payload: Redigera & Försök igen
|
|
168
|
+
edit_payload_confirm: Uppdatera payload och återköa denna händelse?
|
|
169
|
+
edit_payload_label: 'JSON Payload:'
|
|
170
|
+
event_id: 'Händelse-ID:'
|
|
171
|
+
full_json_payload: Fullständig JSON Payload
|
|
172
|
+
headers_section: Headers
|
|
173
|
+
mark_handled: Markera som hanterad
|
|
174
|
+
mark_handled_confirm: Markera denna händelse som hanterad? Den kommer att arkiveras och hoppas över vid uppspelning.
|
|
175
|
+
metadata: Metadata
|
|
176
|
+
metadata_labels:
|
|
177
|
+
last_read: 'Senast läst:'
|
|
178
|
+
read_count: 'Läsantal:'
|
|
179
|
+
visible_at: 'Synlig vid:'
|
|
180
|
+
reroute: Omdirigera
|
|
181
|
+
reroute_confirm: Omdirigera denna händelse till en annan hanterare?
|
|
182
|
+
reroute_label: 'Målhanterare:'
|
|
130
183
|
show:
|
|
131
184
|
back: Tillbaka till händelser
|
|
132
185
|
labels:
|
|
@@ -136,6 +189,9 @@ sv:
|
|
|
136
189
|
not_found: Händelse hittades inte
|
|
137
190
|
title: Händelse %{event_id}
|
|
138
191
|
helpers:
|
|
192
|
+
bulk_select_all: Markera alla
|
|
193
|
+
bulk_select_row: Markera %{id}
|
|
194
|
+
bulk_selected: valda
|
|
139
195
|
paused_badge: Pausad
|
|
140
196
|
queue_badge:
|
|
141
197
|
dlq: DLQ
|
|
@@ -150,11 +206,22 @@ sv:
|
|
|
150
206
|
show:
|
|
151
207
|
charts:
|
|
152
208
|
failed_to_load: Misslyckades att ladda diagramdata
|
|
209
|
+
latency: Köfördröjning (ms)
|
|
210
|
+
latency_avg: Genomsnitt
|
|
211
|
+
latency_p95: P95
|
|
153
212
|
no_data: Inga data än
|
|
154
213
|
series_name: Jobb/min
|
|
155
214
|
status_distribution: Statusfördelning
|
|
156
215
|
throughput: Genomströmning (jobb/min)
|
|
157
216
|
description_html: Jobbprestandamått för de senaste %{range}
|
|
217
|
+
latency_by_queue:
|
|
218
|
+
empty: Ingen fördröjningsdata än
|
|
219
|
+
headers:
|
|
220
|
+
avg: Genomsnitt (ms)
|
|
221
|
+
count: Antal
|
|
222
|
+
p95: P95 (ms)
|
|
223
|
+
queue: Kö
|
|
224
|
+
title: Fördröjning per kö
|
|
158
225
|
slowest:
|
|
159
226
|
empty: Inga jobbstatistik än
|
|
160
227
|
headers:
|
|
@@ -163,11 +230,32 @@ sv:
|
|
|
163
230
|
job_class: Jobbklass
|
|
164
231
|
max: Max
|
|
165
232
|
title: Långsammaste jobbklasser (genomsnittlig varaktighet)
|
|
233
|
+
streams:
|
|
234
|
+
summary:
|
|
235
|
+
active: Aktiv
|
|
236
|
+
avg_fanout: Genomsnittlig spridning
|
|
237
|
+
broadcasts: Sändningar
|
|
238
|
+
connects: Anslutningar
|
|
239
|
+
disconnects: Frånkopplingar
|
|
240
|
+
title: Strömmar i realtid
|
|
241
|
+
top:
|
|
242
|
+
empty: Ingen strömaktivitet registrerad i det valda fönstret
|
|
243
|
+
headers:
|
|
244
|
+
avg_fanout: Genomsnittlig spridning
|
|
245
|
+
avg_ms: Genomsnittlig utsändning
|
|
246
|
+
broadcasts: Sändningar
|
|
247
|
+
stream: Ström
|
|
248
|
+
title: Toppströmmar efter sändningsvolym
|
|
166
249
|
summary:
|
|
167
250
|
avg_duration: Genomsnittlig varaktighet
|
|
251
|
+
avg_latency: Genomsnittlig fördröjning
|
|
252
|
+
avg_retries: Genomsnittliga omförsök
|
|
168
253
|
dead_lettered: Dead Lettered
|
|
169
254
|
failed: Misslyckades
|
|
170
255
|
max_duration: Maximal varaktighet
|
|
256
|
+
p50_latency: P50-fördröjning
|
|
257
|
+
p95_latency: P95-fördröjning
|
|
258
|
+
p99_latency: P99-fördröjning
|
|
171
259
|
succeeded: Lyckades
|
|
172
260
|
total_jobs: Totalt antal jobb
|
|
173
261
|
time_ranges:
|
|
@@ -222,6 +310,12 @@ sv:
|
|
|
222
310
|
discard_all: Kassera alla
|
|
223
311
|
discard_all_confirm: Kassera alla misslyckade jobb?
|
|
224
312
|
discard_all_enqueued_notice: Kasserade %{count} köade jobb och frigjorde deras lås.
|
|
313
|
+
discard_selected: Kassera valda
|
|
314
|
+
discard_selected_confirm: Kassera valda objekt?
|
|
315
|
+
discarded_selected:
|
|
316
|
+
one: Kasserade 1 valt objekt.
|
|
317
|
+
other: Kasserade %{count} valda objekt.
|
|
318
|
+
none_selected: Inga objekt valda.
|
|
225
319
|
retry_all: Försök igen alla
|
|
226
320
|
retry_all_confirm: Försök igen alla misslyckade jobb?
|
|
227
321
|
title: Jobb
|
|
@@ -259,13 +353,28 @@ sv:
|
|
|
259
353
|
toggle_menu: Växla meny
|
|
260
354
|
locks:
|
|
261
355
|
index:
|
|
356
|
+
all_locks_discarded:
|
|
357
|
+
one: Kasserade 1 lås.
|
|
358
|
+
other: Kasserade %{count} lås.
|
|
262
359
|
description: Aktiva unika lås som förhindrar duplicerad jobbexekvering
|
|
360
|
+
discard: Kassera
|
|
361
|
+
discard_all: Kassera alla
|
|
362
|
+
discard_all_confirm: Kassera alla lås permanent? Detta kan tillåta duplicerad jobbexekvering.
|
|
363
|
+
discard_confirm: Kassera detta lås? Det associerade jobbet kan köas igen.
|
|
364
|
+
discard_selected: Kassera valda
|
|
365
|
+
discard_selected_confirm: Kassera valda lås?
|
|
263
366
|
empty: Inga aktiva lås
|
|
264
367
|
headers:
|
|
265
368
|
age: Ålder
|
|
266
369
|
lock_key: Låsningsnyckel
|
|
267
370
|
msg_id: Meddelande-ID
|
|
268
371
|
queue_name: Kö
|
|
372
|
+
lock_discard_failed: Kunde inte kassera lås.
|
|
373
|
+
lock_discarded: Lås kasserat.
|
|
374
|
+
locks_discarded:
|
|
375
|
+
one: Kasserade 1 lås.
|
|
376
|
+
other: Kasserade %{count} lås.
|
|
377
|
+
none_selected: Inga lås valda.
|
|
269
378
|
title: Unikhetsnycklar
|
|
270
379
|
outbox:
|
|
271
380
|
index:
|
data/config/routes.rb
CHANGED
|
@@ -156,6 +156,11 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
|
|
|
156
156
|
# queue processing and concurrency lock management.
|
|
157
157
|
execute Pgbus::AutovacuumTuning.sql_for_all_queues
|
|
158
158
|
execute Pgbus::AutovacuumTuning.sql_for_high_churn_tables
|
|
159
|
+
|
|
160
|
+
# Set fillfactor on queue tables to reduce bloat from PGMQ's read
|
|
161
|
+
# UPDATE operations (vt, read_ct, last_read_at). Lower fillfactor
|
|
162
|
+
# reserves page space, reducing page density during heavy update churn.
|
|
163
|
+
execute Pgbus::TableMaintenance.fillfactor_sql_for_all_queues
|
|
159
164
|
end
|
|
160
165
|
|
|
161
166
|
def down
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
class TunePgbusFillfactor < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def up
|
|
3
|
+
# Set fillfactor on queue tables to reduce bloat from PGMQ's read
|
|
4
|
+
# UPDATE operations. PGMQ updates vt, read_ct, and last_read_at on
|
|
5
|
+
# every read — with fillfactor=100 (default), pages fill completely
|
|
6
|
+
# between vacuum passes. Lowering fillfactor reserves page space,
|
|
7
|
+
# reducing page density during heavy update churn. Note: because vt
|
|
8
|
+
# is indexed, these updates are not HOT-eligible.
|
|
9
|
+
#
|
|
10
|
+
# Archive tables are append-only (INSERT from queue, DELETE on
|
|
11
|
+
# retention) and don't benefit from fillfactor tuning.
|
|
12
|
+
#
|
|
13
|
+
# New queues created after this migration automatically receive
|
|
14
|
+
# this setting via Pgbus::Client at queue creation time.
|
|
15
|
+
execute Pgbus::TableMaintenance.fillfactor_sql_for_all_queues
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def down
|
|
19
|
+
execute <<~SQL
|
|
20
|
+
DO $$
|
|
21
|
+
DECLARE
|
|
22
|
+
q RECORD;
|
|
23
|
+
BEGIN
|
|
24
|
+
FOR q IN SELECT queue_name FROM pgmq.meta LOOP
|
|
25
|
+
EXECUTE format('ALTER TABLE pgmq.q_%I RESET (fillfactor)', q.queue_name);
|
|
26
|
+
END LOOP;
|
|
27
|
+
END $$;
|
|
28
|
+
SQL
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
require_relative "migration_path"
|
|
6
|
+
|
|
7
|
+
module Pgbus
|
|
8
|
+
module Generators
|
|
9
|
+
class TuneFillfactorGenerator < Rails::Generators::Base
|
|
10
|
+
include ActiveRecord::Generators::Migration
|
|
11
|
+
include MigrationPath
|
|
12
|
+
|
|
13
|
+
source_root File.expand_path("templates", __dir__)
|
|
14
|
+
|
|
15
|
+
desc "Set fillfactor on PGMQ queue tables to reduce page density and bloat"
|
|
16
|
+
|
|
17
|
+
class_option :database,
|
|
18
|
+
type: :string,
|
|
19
|
+
default: nil,
|
|
20
|
+
desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
|
|
21
|
+
|
|
22
|
+
def create_migration_file
|
|
23
|
+
migration_template "tune_fillfactor.rb.erb",
|
|
24
|
+
File.join(pgbus_migrate_path, "tune_pgbus_fillfactor.rb")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def display_post_install
|
|
28
|
+
say ""
|
|
29
|
+
say "Pgbus fillfactor tuning migration created!", :green
|
|
30
|
+
say ""
|
|
31
|
+
say "This migration sets fillfactor=#{Pgbus::TableMaintenance::FILLFACTOR} on all existing"
|
|
32
|
+
say "PGMQ queue tables. This reserves #{100 - Pgbus::TableMaintenance::FILLFACTOR}% of each page to"
|
|
33
|
+
say "reduce page density during PGMQ's heavy read UPDATE churn."
|
|
34
|
+
say ""
|
|
35
|
+
say "New queues created at runtime will automatically receive"
|
|
36
|
+
say "this setting."
|
|
37
|
+
say ""
|
|
38
|
+
say "Next steps:"
|
|
39
|
+
say " 1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
|
|
40
|
+
say " 2. Restart pgbus: bin/pgbus start"
|
|
41
|
+
say ""
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def migration_version
|
|
47
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
data/lib/pgbus/client.rb
CHANGED
|
@@ -87,7 +87,9 @@ module Pgbus
|
|
|
87
87
|
target = @queue_strategy.target_queue(queue_name, priority)
|
|
88
88
|
ensure_queue(queue_name)
|
|
89
89
|
Instrumentation.instrument("pgbus.client.send_message", queue: target) do
|
|
90
|
-
|
|
90
|
+
with_stale_connection_retry do
|
|
91
|
+
synchronized { @pgmq.produce(target, serialize(payload), headers: headers && serialize(headers), delay: delay) }
|
|
92
|
+
end
|
|
91
93
|
end
|
|
92
94
|
end
|
|
93
95
|
|
|
@@ -96,7 +98,9 @@ module Pgbus
|
|
|
96
98
|
ensure_queue(queue_name)
|
|
97
99
|
serialized, serialized_headers = serialize_batch(payloads, headers)
|
|
98
100
|
Instrumentation.instrument("pgbus.client.send_batch", queue: full_name, size: payloads.size) do
|
|
99
|
-
|
|
101
|
+
with_stale_connection_retry do
|
|
102
|
+
synchronized { @pgmq.produce_batch(full_name, serialized, headers: serialized_headers, delay: delay) }
|
|
103
|
+
end
|
|
100
104
|
end
|
|
101
105
|
end
|
|
102
106
|
|
|
@@ -318,13 +322,15 @@ module Pgbus
|
|
|
318
322
|
end
|
|
319
323
|
|
|
320
324
|
def publish_to_topic(routing_key, payload, headers: nil, delay: 0)
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
325
|
+
with_stale_connection_retry do
|
|
326
|
+
synchronized do
|
|
327
|
+
@pgmq.produce_topic(
|
|
328
|
+
routing_key,
|
|
329
|
+
serialize(payload),
|
|
330
|
+
headers: headers && serialize(headers),
|
|
331
|
+
delay: delay
|
|
332
|
+
)
|
|
333
|
+
end
|
|
328
334
|
end
|
|
329
335
|
end
|
|
330
336
|
|
|
@@ -502,6 +508,7 @@ module Pgbus
|
|
|
502
508
|
def tune_autovacuum(queue_name)
|
|
503
509
|
with_raw_connection do |conn|
|
|
504
510
|
conn.exec(AutovacuumTuning.sql_for_queue(queue_name))
|
|
511
|
+
conn.exec(TableMaintenance.fillfactor_sql_for_queue(queue_name))
|
|
505
512
|
end
|
|
506
513
|
rescue StandardError => e
|
|
507
514
|
Pgbus.logger.debug { "[Pgbus::Client] Autovacuum tuning failed for #{queue_name}: #{e.message}" }
|
|
@@ -518,6 +525,51 @@ module Pgbus
|
|
|
518
525
|
end
|
|
519
526
|
end
|
|
520
527
|
|
|
528
|
+
# Substrings that indicate the pooled PG::Connection was already dead
|
|
529
|
+
# *before* pgmq-ruby tried to use it — typically killed by a connection
|
|
530
|
+
# pooler (PgBouncer server_idle_timeout / client_idle_timeout), an admin
|
|
531
|
+
# disconnect, or a TCP RST while the slot was idle.
|
|
532
|
+
#
|
|
533
|
+
# Only pre-checkout / pre-flight errors belong here. Mid-flight errors
|
|
534
|
+
# like "server closed the connection" or "connection to server was lost"
|
|
535
|
+
# are excluded because PG may have already committed the INSERT before
|
|
536
|
+
# the socket died, and retrying would duplicate the message.
|
|
537
|
+
#
|
|
538
|
+
# See mensfeld/pgmq-ruby#94.
|
|
539
|
+
STALE_CONNECTION_PATTERNS = [
|
|
540
|
+
"pqsocket() can't get socket descriptor",
|
|
541
|
+
"connection is closed",
|
|
542
|
+
"connection has been closed",
|
|
543
|
+
"connection not open",
|
|
544
|
+
"no connection to the server"
|
|
545
|
+
].freeze
|
|
546
|
+
private_constant :STALE_CONNECTION_PATTERNS
|
|
547
|
+
|
|
548
|
+
# Enqueue path guard: rescue PGMQ::Errors::ConnectionError once if its
|
|
549
|
+
# message matches a known stale-socket pattern. pgmq-ruby's
|
|
550
|
+
# auto_reconnect + verify_connection! already recovers on the *next*
|
|
551
|
+
# checkout, so a single retry is sufficient. Other connection errors
|
|
552
|
+
# (pool timeout, misconfiguration, truly unreachable DB) propagate.
|
|
553
|
+
def with_stale_connection_retry
|
|
554
|
+
attempts = 0
|
|
555
|
+
begin
|
|
556
|
+
yield
|
|
557
|
+
rescue PGMQ::Errors::ConnectionError => e
|
|
558
|
+
attempts += 1
|
|
559
|
+
raise unless attempts == 1 && stale_connection_error?(e)
|
|
560
|
+
|
|
561
|
+
Pgbus.logger.warn do
|
|
562
|
+
"[Pgbus::Client] Retrying produce after stale pgmq connection: #{e.message}"
|
|
563
|
+
end
|
|
564
|
+
retry
|
|
565
|
+
end
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
def stale_connection_error?(error)
|
|
569
|
+
message = error.message.to_s.downcase
|
|
570
|
+
STALE_CONNECTION_PATTERNS.any? { |pattern| message.include?(pattern) }
|
|
571
|
+
end
|
|
572
|
+
|
|
521
573
|
def serialize(data)
|
|
522
574
|
case data
|
|
523
575
|
when String
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -78,6 +78,7 @@ module Pgbus
|
|
|
78
78
|
|
|
79
79
|
# Recurring jobs
|
|
80
80
|
attr_accessor :recurring_tasks, :recurring_schedule_interval, :recurring_tasks_file, :skip_recurring
|
|
81
|
+
attr_writer :recurring_tasks_files
|
|
81
82
|
attr_reader :recurring_execution_retention # rubocop:disable Style/AccessorGrouping
|
|
82
83
|
|
|
83
84
|
# Multi-database support (optional separate database for pgbus tables)
|
|
@@ -161,6 +162,7 @@ module Pgbus
|
|
|
161
162
|
@recurring_tasks = nil
|
|
162
163
|
@recurring_schedule_interval = 1.0
|
|
163
164
|
@recurring_tasks_file = nil
|
|
165
|
+
@recurring_tasks_files = nil
|
|
164
166
|
@skip_recurring = false
|
|
165
167
|
@recurring_execution_retention = 7 * 24 * 3600 # 7 days
|
|
166
168
|
|
|
@@ -492,6 +494,12 @@ module Pgbus
|
|
|
492
494
|
@recurring_execution_retention = coerce_duration!(value, :recurring_execution_retention)
|
|
493
495
|
end
|
|
494
496
|
|
|
497
|
+
def recurring_tasks_files
|
|
498
|
+
return @recurring_tasks_files if @recurring_tasks_files
|
|
499
|
+
|
|
500
|
+
recurring_tasks_file ? [recurring_tasks_file] : nil
|
|
501
|
+
end
|
|
502
|
+
|
|
495
503
|
# Returns the connection pool size to use for the PGMQ client.
|
|
496
504
|
#
|
|
497
505
|
# If +pool_size+ was explicitly set, returns that value unchanged. Otherwise
|
data/lib/pgbus/engine.rb
CHANGED
|
@@ -18,10 +18,22 @@ module Pgbus
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
initializer "pgbus.recurring" do |app|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
21
|
+
next if Pgbus.configuration.recurring_tasks
|
|
22
|
+
|
|
23
|
+
config = Pgbus.configuration
|
|
24
|
+
files = config.recurring_tasks_files
|
|
25
|
+
default_path = app.root.join("config", "recurring.yml")
|
|
26
|
+
|
|
27
|
+
if files
|
|
28
|
+
tasks = Pgbus::Recurring::ConfigLoader.load_all(files)
|
|
29
|
+
if tasks.empty? && default_path.exist? && files.none? { |f| File.expand_path(f.to_s) == File.expand_path(default_path.to_s) }
|
|
30
|
+
tasks = Pgbus::Recurring::ConfigLoader.load(default_path)
|
|
31
|
+
config.recurring_tasks_file ||= default_path.to_s
|
|
32
|
+
end
|
|
33
|
+
config.recurring_tasks = tasks unless tasks.empty?
|
|
34
|
+
elsif default_path.exist?
|
|
35
|
+
config.recurring_tasks = Pgbus::Recurring::ConfigLoader.load(default_path)
|
|
36
|
+
config.recurring_tasks_file ||= default_path.to_s
|
|
25
37
|
end
|
|
26
38
|
end
|
|
27
39
|
|
|
@@ -8,7 +8,7 @@ module Pgbus
|
|
|
8
8
|
module_function
|
|
9
9
|
|
|
10
10
|
def publish(routing_key, payload, headers: nil, delay: 0)
|
|
11
|
-
event_data = build_event_data(payload)
|
|
11
|
+
event_data = build_event_data(payload, routing_key: routing_key)
|
|
12
12
|
|
|
13
13
|
if defined?(Pgbus::Testing) && !Pgbus::Testing.disabled?
|
|
14
14
|
event = Pgbus::Event.new(
|
|
@@ -37,7 +37,7 @@ module Pgbus
|
|
|
37
37
|
publish(routing_key, payload, headers: headers, delay: delay)
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
-
def build_event_data(payload)
|
|
40
|
+
def build_event_data(payload, routing_key: nil)
|
|
41
41
|
event_id = SecureRandom.uuid
|
|
42
42
|
|
|
43
43
|
serialized_payload = if payload.respond_to?(:to_global_id)
|
|
@@ -48,11 +48,13 @@ module Pgbus
|
|
|
48
48
|
{ "value" => payload }
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
{
|
|
51
|
+
data = {
|
|
52
52
|
"event_id" => event_id,
|
|
53
53
|
"payload" => serialized_payload,
|
|
54
54
|
"published_at" => Time.now.utc.iso8601(6)
|
|
55
55
|
}
|
|
56
|
+
data["routing_key"] = routing_key if routing_key
|
|
57
|
+
data
|
|
56
58
|
end
|
|
57
59
|
end
|
|
58
60
|
end
|