pgbus 0.0.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +37 -3
- data/Rakefile +98 -1
- data/app/controllers/pgbus/application_controller.rb +8 -0
- data/app/controllers/pgbus/recurring_tasks_controller.rb +36 -0
- data/app/helpers/pgbus/application_helper.rb +41 -0
- data/app/models/pgbus/application_record.rb +7 -0
- data/app/models/pgbus/batch_entry.rb +31 -0
- data/app/models/pgbus/blocked_execution.rb +40 -0
- data/app/models/pgbus/process_entry.rb +9 -0
- data/app/models/pgbus/processed_event.rb +9 -0
- data/app/models/pgbus/recurring_execution.rb +33 -0
- data/app/models/pgbus/recurring_task.rb +42 -0
- data/app/models/pgbus/semaphore.rb +29 -0
- data/app/views/layouts/pgbus/application.html.erb +1 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +9 -1
- data/app/views/pgbus/dead_letter/_messages_table.html.erb +55 -18
- data/app/views/pgbus/jobs/_enqueued_table.html.erb +46 -8
- data/app/views/pgbus/recurring_tasks/_tasks_table.html.erb +79 -0
- data/app/views/pgbus/recurring_tasks/index.html.erb +6 -0
- data/app/views/pgbus/recurring_tasks/show.html.erb +122 -0
- data/config/routes.rb +7 -0
- data/lib/active_job/queue_adapters/pgbus_adapter.rb +29 -0
- data/lib/generators/pgbus/add_recurring_generator.rb +56 -0
- data/lib/generators/pgbus/install_generator.rb +76 -2
- data/lib/generators/pgbus/templates/add_recurring_tables.rb.erb +31 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +72 -4
- data/lib/generators/pgbus/templates/recurring.yml.erb +40 -0
- data/lib/generators/pgbus/templates/upgrade_pgmq.rb.erb +30 -0
- data/lib/generators/pgbus/upgrade_pgmq_generator.rb +60 -0
- data/lib/pgbus/active_job/adapter.rb +3 -6
- data/lib/pgbus/active_job/executor.rb +26 -12
- data/lib/pgbus/batch.rb +65 -72
- data/lib/pgbus/cli.rb +11 -16
- data/lib/pgbus/client.rb +32 -15
- data/lib/pgbus/concurrency/blocked_execution.rb +32 -37
- data/lib/pgbus/concurrency/semaphore.rb +11 -39
- data/lib/pgbus/concurrency.rb +10 -2
- data/lib/pgbus/configuration.rb +48 -0
- data/lib/pgbus/engine.rb +19 -1
- data/lib/pgbus/event_bus/handler.rb +10 -23
- data/lib/pgbus/instrumentation.rb +29 -0
- data/lib/pgbus/pgmq_schema/pgmq_v1.11.0.sql +2123 -0
- data/lib/pgbus/pgmq_schema.rb +159 -0
- data/lib/pgbus/process/consumer.rb +17 -9
- data/lib/pgbus/process/dispatcher.rb +33 -41
- data/lib/pgbus/process/heartbeat.rb +15 -23
- data/lib/pgbus/process/signal_handler.rb +23 -1
- data/lib/pgbus/process/supervisor.rb +79 -2
- data/lib/pgbus/process/worker.rb +42 -13
- data/lib/pgbus/recurring/already_recorded.rb +7 -0
- data/lib/pgbus/recurring/command_job.rb +28 -0
- data/lib/pgbus/recurring/config_loader.rb +35 -0
- data/lib/pgbus/recurring/schedule.rb +102 -0
- data/lib/pgbus/recurring/scheduler.rb +102 -0
- data/lib/pgbus/recurring/task.rb +111 -0
- data/lib/pgbus/serializer.rb +16 -6
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +217 -36
- data/lib/pgbus.rb +8 -0
- data/lib/tasks/pgbus_pgmq.rake +62 -0
- metadata +51 -24
- data/.bun-version +0 -1
- data/.claude/commands/architect.md +0 -100
- data/.claude/commands/github-review-comments.md +0 -237
- data/.claude/commands/lfg.md +0 -271
- data/.claude/commands/review-pr.md +0 -69
- data/.claude/commands/security.md +0 -122
- data/.claude/commands/tdd.md +0 -148
- data/.claude/rules/agents.md +0 -49
- data/.claude/rules/coding-style.md +0 -91
- data/.claude/rules/git-workflow.md +0 -56
- data/.claude/rules/performance.md +0 -73
- data/.claude/rules/testing.md +0 -67
- data/CLAUDE.md +0 -80
- data/CODE_OF_CONDUCT.md +0 -10
- data/bun.lock +0 -18
- data/docs/README.md +0 -28
- data/docs/switch_from_good_job.md +0 -279
- data/docs/switch_from_sidekiq.md +0 -226
- data/docs/switch_from_solid_queue.md +0 -247
- data/package.json +0 -9
- data/sig/pgbus.rbs +0 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2aad79af8c595a48d879b9bad0ebaaf43d40843cdbcb9dcc7a2dad0023257e53
|
|
4
|
+
data.tar.gz: 53bda3ba2e7d1d0f1935d183193bff5b48adeca8a4da44b6d55f9ba743e62d8e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c9cfcd463d6b91591f9f037969ca1b30c2dc29ba0ce7a8eeefa829ef2b12eee71c4aba0f5b8d02b3336d6cc38b04db7cc51d749ca97774d4afa4e2246f8d6ccc
|
|
7
|
+
data.tar.gz: a50591b61b5bf49a2b5e14750622ba7ca0681d3db5f47d801542a23eb3147a8b9d5a605fcd0c55264ad4068e75f3d827ad550d2d58cd2026fa8a4351f72eea8a
|
data/README.md
CHANGED
|
@@ -204,8 +204,6 @@ Limit how many jobs with the same key can run concurrently:
|
|
|
204
204
|
|
|
205
205
|
```ruby
|
|
206
206
|
class ProcessOrderJob < ApplicationJob
|
|
207
|
-
include Pgbus::Concurrency
|
|
208
|
-
|
|
209
207
|
limits_concurrency to: 1,
|
|
210
208
|
key: ->(order_id) { "ProcessOrder-#{order_id}" },
|
|
211
209
|
duration: 15.minutes,
|
|
@@ -237,9 +235,45 @@ end
|
|
|
237
235
|
### How it works
|
|
238
236
|
|
|
239
237
|
1. **Enqueue**: The adapter checks a semaphore table for the concurrency key. If under the limit, it increments the counter and sends the job to PGMQ. If at the limit, it applies the `on_conflict` strategy.
|
|
240
|
-
2. **Complete**: After a job succeeds or is dead-lettered, the executor
|
|
238
|
+
2. **Complete**: After a job succeeds or is dead-lettered, the executor signals the concurrency system via an `ensure` block (guaranteeing the signal fires even if the archive step fails). It first tries to promote a blocked job (atomic delete + enqueue in a single transaction). If nothing to promote, it releases the semaphore slot.
|
|
241
239
|
3. **Safety net**: The dispatcher periodically cleans up expired semaphores and orphaned blocked executions to recover from crashed workers.
|
|
242
240
|
|
|
241
|
+
### Concurrency compared to other backends
|
|
242
|
+
|
|
243
|
+
Pgbus, SolidQueue, GoodJob, and Sidekiq all offer concurrency controls, but with fundamentally different locking strategies and trade-offs.
|
|
244
|
+
|
|
245
|
+
#### Architecture comparison
|
|
246
|
+
|
|
247
|
+
| | **Pgbus** | **SolidQueue** | **GoodJob** | **Sidekiq Enterprise** |
|
|
248
|
+
|---|---|---|---|---|
|
|
249
|
+
| **Lock backend** | PostgreSQL rows (`pgbus_semaphores` table) | PostgreSQL rows (`solid_queue_semaphores`) | PostgreSQL advisory locks (`pg_advisory_xact_lock`) | Redis sorted sets (lease-based) |
|
|
250
|
+
| **Lock granularity** | Counting semaphore (allows N concurrent) | Counting semaphore (allows N concurrent) | Count query under advisory lock | Sorted set entries with TTL |
|
|
251
|
+
| **Acquire mechanism** | Atomic `INSERT ... ON CONFLICT DO UPDATE WHERE value < max` (single SQL) | `UPDATE ... SET value = value + 1 WHERE value < limit` | `pg_advisory_xact_lock` then `SELECT COUNT(*)` in rolled-back txn | Redis Lua script (atomic check-and-add) |
|
|
252
|
+
| **At-limit behavior** | `:block` (hold in queue), `:discard`, or `:raise` | Blocks in `solid_queue_blocked_executions` | Enqueue: silently dropped. Perform: retry with backoff (forever) | Reschedule with backoff (raises `OverLimit`, middleware re-enqueues) |
|
|
253
|
+
| **Blocked job storage** | `pgbus_blocked_executions` table with priority ordering | `solid_queue_blocked_executions` table | No blocked queue — retries via ActiveJob retry mechanism | No blocked queue — job returns to Redis queue with delay |
|
|
254
|
+
| **Release on completion** | `ensure` block: promote next blocked job or decrement semaphore | Inline after `finished`/`failed_with` (inside same transaction as of PR #689) | Release advisory lock via `pg_advisory_unlock` | Lease auto-expires from sorted set |
|
|
255
|
+
| **Crash recovery** | Semaphore `expires_at` + dispatcher `expire_stale` cleanup | Semaphore `expires_at` + concurrency maintenance task | Advisory locks auto-release on session disconnect | TTL-based lease expiry (default 5 min) |
|
|
256
|
+
| **Message lifecycle** | PGMQ visibility timeout (`FOR UPDATE SKIP LOCKED`) — message stays in queue until archived | AR-backed `claimed_executions` table | AR-backed `good_jobs` table with advisory lock per row | Redis list + sorted set |
|
|
257
|
+
|
|
258
|
+
#### Key design differences
|
|
259
|
+
|
|
260
|
+
**Pgbus** uses PGMQ's native `FOR UPDATE SKIP LOCKED` for message claiming and a separate semaphore table for concurrency control. This two-layer approach means the message queue and concurrency system are independent — PGMQ handles exactly-once delivery, the semaphore handles admission control. The semaphore acquire is a single atomic SQL (`INSERT ... ON CONFLICT DO UPDATE WHERE value < max`), avoiding the need for explicit row locks.
|
|
261
|
+
|
|
262
|
+
**SolidQueue** uses AR models for everything — jobs, claimed executions, and semaphores all live in PostgreSQL tables. This means the entire lifecycle can be wrapped in AR transactions. However, as documented in [rails/solid_queue#689](https://github.com/rails/solid_queue/pull/689), this model is vulnerable to race conditions when semaphore expiry, job completion, and blocked-job release interleave across transactions. Pgbus avoids several of these by design: PGMQ's visibility timeout handles message recovery without a `claimed_executions` table, and there is no "release during shutdown" codepath.
|
|
263
|
+
|
|
264
|
+
**GoodJob** takes a different approach entirely: advisory locks. Each job dequeue acquires a session-level advisory lock on the job row, and concurrency checks use transaction-scoped advisory locks on the concurrency key. This means the check and the perform are serialized at the database level. The downside is that advisory locks are session-scoped — if a connection is returned to the pool without unlocking, the lock persists. GoodJob handles this by auto-releasing on session disconnect, but connection pool sharing between web and worker can cause surprising behavior.
|
|
265
|
+
|
|
266
|
+
**Sidekiq Enterprise** uses Redis sorted sets with TTL-based leases. Each concurrent slot is a sorted set entry with an expiry timestamp. This is fast and simple but has no durability guarantee — Redis failover can lose leases, temporarily allowing over-limit execution. The `sidekiq-unique-jobs` gem (open-source) uses a similar Lua-script approach but with more lock strategies (`:until_executing`, `:while_executing`, `:until_and_while_executing`) and configurable conflict handlers (`:reject`, `:reschedule`, `:replace`, `:raise`).
|
|
267
|
+
|
|
268
|
+
#### Race condition resilience
|
|
269
|
+
|
|
270
|
+
| Scenario | Pgbus | SolidQueue | GoodJob | Sidekiq |
|
|
271
|
+
|---|---|---|---|---|
|
|
272
|
+
| **Worker crash mid-execution** | PGMQ visibility timeout expires → message re-read. Semaphore expires via `expire_stale`. | `claimed_execution` survives → supervisor's process pruning calls `fail_all_with`. | Advisory lock released on session disconnect. | Lease TTL expires in Redis. |
|
|
273
|
+
| **Blocked job released while original still executing** | Not possible — promote only happens in `signal_concurrency`, which only runs after job success/DLQ. | Fixed in PR #689 — now checks for claimed executions before releasing. | N/A — no blocked queue; retries independently. | N/A — no blocked queue. |
|
|
274
|
+
| **Archive succeeds but signal fails** | `ensure` block guarantees signal fires even if archive raises. For SIGKILL: semaphore expires via dispatcher. | Fixed in PR #689 — `unblock_next_job` moved inside same transaction as `finished`. | Advisory lock released by session disconnect. | Lease auto-expires. |
|
|
275
|
+
| **Concurrent enqueue and signal race** | Semaphore acquire is a single atomic SQL — no read-then-write gap. | Fixed in PR #689 — `FOR UPDATE` lock on semaphore row serializes enqueue with signal. | `pg_advisory_xact_lock` serializes the concurrency check. | Redis Lua script is atomic. |
|
|
276
|
+
|
|
243
277
|
## Batches
|
|
244
278
|
|
|
245
279
|
Coordinate groups of jobs with callbacks when all complete:
|
data/Rakefile
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "bundler/gem_tasks"
|
|
4
3
|
require "rspec/core/rake_task"
|
|
5
4
|
|
|
6
5
|
RSpec::Core::RakeTask.new(:spec) do |t|
|
|
@@ -11,4 +10,102 @@ require "rubocop/rake_task"
|
|
|
11
10
|
|
|
12
11
|
RuboCop::RakeTask.new
|
|
13
12
|
|
|
13
|
+
namespace :bench do
|
|
14
|
+
desc "Run serialization benchmarks"
|
|
15
|
+
task :serialization do
|
|
16
|
+
ruby "benchmarks/serialization_bench.rb"
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
desc "Run client operation benchmarks"
|
|
20
|
+
task :client do
|
|
21
|
+
ruby "benchmarks/client_bench.rb"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
desc "Run executor benchmarks"
|
|
25
|
+
task :executor do
|
|
26
|
+
ruby "benchmarks/executor_bench.rb"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
desc "Run detailed memory profiling"
|
|
30
|
+
task :memory do
|
|
31
|
+
ruby "benchmarks/memory_profile.rb"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
desc "Run all benchmarks"
|
|
35
|
+
task all: %i[serialization client executor]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
desc "Run all benchmarks"
|
|
39
|
+
task bench: "bench:all"
|
|
40
|
+
|
|
41
|
+
desc "Build gem and verify contents"
|
|
42
|
+
task :build do
|
|
43
|
+
sh("gem build pgbus.gemspec --strict")
|
|
44
|
+
gem_file = Dir["pgbus-*.gem"].first
|
|
45
|
+
abort "Gem file not found after build" unless gem_file
|
|
46
|
+
|
|
47
|
+
sh("gem unpack #{gem_file} --target /tmp/gem-verify")
|
|
48
|
+
puts "\n=== Gem contents ==="
|
|
49
|
+
sh("find /tmp/gem-verify -type f | sort")
|
|
50
|
+
sh("rm -rf /tmp/gem-verify #{gem_file}")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
desc "Release a new version (rake release[1.2.3] or rake release[pre])"
|
|
54
|
+
task :release, [:version] do |_t, args|
|
|
55
|
+
require_relative "lib/pgbus/version"
|
|
56
|
+
|
|
57
|
+
new_version = args[:version]
|
|
58
|
+
abort "Usage: rake release[X.Y.Z] or rake release[pre]" unless new_version
|
|
59
|
+
|
|
60
|
+
dirty = `git status --porcelain`.strip
|
|
61
|
+
abort "Aborting: working directory is not clean.\n#{dirty}" unless dirty.empty?
|
|
62
|
+
|
|
63
|
+
current = Pgbus::VERSION
|
|
64
|
+
prerelease = new_version.match?(/alpha|beta|rc|pre/) || new_version == "pre"
|
|
65
|
+
|
|
66
|
+
if new_version == "pre"
|
|
67
|
+
new_version = current
|
|
68
|
+
prerelease = true
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
tag = "v#{new_version}"
|
|
72
|
+
|
|
73
|
+
puts "Current version: #{current}"
|
|
74
|
+
puts "New version: #{new_version}"
|
|
75
|
+
puts "Tag: #{tag}"
|
|
76
|
+
puts "Pre-release: #{prerelease}"
|
|
77
|
+
puts ""
|
|
78
|
+
|
|
79
|
+
# Update version file if needed
|
|
80
|
+
version_file = "lib/pgbus/version.rb"
|
|
81
|
+
if new_version != current
|
|
82
|
+
content = File.read(version_file)
|
|
83
|
+
content.sub!(/VERSION = ".*"/, "VERSION = \"#{new_version}\"")
|
|
84
|
+
File.write(version_file, content)
|
|
85
|
+
puts "Updated #{version_file}"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Verify gem builds cleanly
|
|
89
|
+
sh("gem build pgbus.gemspec --strict")
|
|
90
|
+
sh("rm -f pgbus-*.gem")
|
|
91
|
+
|
|
92
|
+
# Commit, push, and create release
|
|
93
|
+
if new_version != current
|
|
94
|
+
sh("git add #{version_file}")
|
|
95
|
+
sh("git commit -m 'chore: bump version to #{new_version}'")
|
|
96
|
+
end
|
|
97
|
+
sh("git push origin main")
|
|
98
|
+
|
|
99
|
+
pre_flag = prerelease ? "--prerelease" : ""
|
|
100
|
+
sh("gh release create #{tag} --generate-notes --target main #{pre_flag}".strip)
|
|
101
|
+
|
|
102
|
+
puts ""
|
|
103
|
+
puts "Release #{tag} created! CI will handle the rest:"
|
|
104
|
+
puts " - Run tests"
|
|
105
|
+
puts " - Build + verify gem"
|
|
106
|
+
puts " - Sign with Sigstore"
|
|
107
|
+
puts " - Publish to RubyGems"
|
|
108
|
+
puts " - Upload assets to the release"
|
|
109
|
+
end
|
|
110
|
+
|
|
14
111
|
task default: %i[spec rubocop]
|
|
@@ -10,6 +10,14 @@ module Pgbus
|
|
|
10
10
|
|
|
11
11
|
helper Pgbus::ApplicationHelper
|
|
12
12
|
|
|
13
|
+
# Make `pgbus` route proxy available in views (e.g. pgbus.root_path).
|
|
14
|
+
# With isolate_namespace, the non-prefixed helpers (root_path) work inside
|
|
15
|
+
# the engine, but the views use the pgbus.* proxy form for clarity.
|
|
16
|
+
def pgbus
|
|
17
|
+
@pgbus ||= Pgbus::Engine.routes.url_helpers
|
|
18
|
+
end
|
|
19
|
+
helper_method :pgbus
|
|
20
|
+
|
|
13
21
|
private
|
|
14
22
|
|
|
15
23
|
def data_source
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class RecurringTasksController < ApplicationController
|
|
5
|
+
def index
|
|
6
|
+
case params[:frame]
|
|
7
|
+
when "recurring_tasks"
|
|
8
|
+
@recurring_tasks = data_source.recurring_tasks
|
|
9
|
+
render_frame("pgbus/recurring_tasks/tasks_table")
|
|
10
|
+
else
|
|
11
|
+
@recurring_tasks = data_source.recurring_tasks
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def show
|
|
16
|
+
@task = data_source.recurring_task(params[:id])
|
|
17
|
+
redirect_to pgbus.recurring_tasks_path, alert: "Task not found" unless @task
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def toggle
|
|
21
|
+
if data_source.toggle_recurring_task(params[:id])
|
|
22
|
+
redirect_to pgbus.recurring_tasks_path, notice: "Task toggled"
|
|
23
|
+
else
|
|
24
|
+
redirect_to pgbus.recurring_tasks_path, alert: "Failed to toggle task"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def enqueue
|
|
29
|
+
if data_source.enqueue_recurring_task_now(params[:id])
|
|
30
|
+
redirect_to pgbus.recurring_tasks_path, notice: "Task enqueued"
|
|
31
|
+
else
|
|
32
|
+
redirect_to pgbus.recurring_tasks_path, alert: "Failed to enqueue task"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -45,6 +45,18 @@ module Pgbus
|
|
|
45
45
|
end
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
+
def pgbus_parse_message(message)
|
|
49
|
+
return {} unless message
|
|
50
|
+
|
|
51
|
+
case message
|
|
52
|
+
when Hash then message
|
|
53
|
+
when String then JSON.parse(message)
|
|
54
|
+
else {}
|
|
55
|
+
end
|
|
56
|
+
rescue JSON::ParserError
|
|
57
|
+
{}
|
|
58
|
+
end
|
|
59
|
+
|
|
48
60
|
def pgbus_json_preview(json_string, max_length: 120)
|
|
49
61
|
return "—" unless json_string
|
|
50
62
|
|
|
@@ -56,6 +68,35 @@ module Pgbus
|
|
|
56
68
|
Pgbus.configuration.web_refresh_interval
|
|
57
69
|
end
|
|
58
70
|
|
|
71
|
+
def pgbus_time_ago_future(time)
|
|
72
|
+
return "—" unless time
|
|
73
|
+
|
|
74
|
+
time = Time.parse(time) if time.is_a?(String)
|
|
75
|
+
seconds = (time - Time.now).to_i
|
|
76
|
+
|
|
77
|
+
if seconds <= 0
|
|
78
|
+
"now"
|
|
79
|
+
elsif seconds < 60
|
|
80
|
+
"in #{seconds}s"
|
|
81
|
+
elsif seconds < 3600
|
|
82
|
+
"in #{seconds / 60}m"
|
|
83
|
+
elsif seconds < 86_400
|
|
84
|
+
"in #{seconds / 3600}h"
|
|
85
|
+
else
|
|
86
|
+
"in #{seconds / 86_400}d"
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def pgbus_recurring_health_badge(task)
|
|
91
|
+
if task[:last_run_at].nil?
|
|
92
|
+
tag.span("Pending",
|
|
93
|
+
class: "inline-flex items-center rounded-full bg-yellow-100 px-2.5 py-0.5 text-xs font-medium text-yellow-800")
|
|
94
|
+
else
|
|
95
|
+
tag.span("Active",
|
|
96
|
+
class: "inline-flex items-center rounded-full bg-green-100 px-2.5 py-0.5 text-xs font-medium text-green-800")
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
59
100
|
def pgbus_nav_link(label, path)
|
|
60
101
|
active = request.path == path || (path != pgbus.root_path && request.path.start_with?(path))
|
|
61
102
|
css = if active
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class BatchEntry < ApplicationRecord
|
|
5
|
+
self.table_name = "pgbus_batches"
|
|
6
|
+
|
|
7
|
+
COUNTER_COLUMNS = %w[completed_jobs discarded_jobs].freeze
|
|
8
|
+
|
|
9
|
+
scope :finished, -> { where(status: "finished") }
|
|
10
|
+
scope :stale, ->(before:) { finished.where("finished_at < ?", before) }
|
|
11
|
+
|
|
12
|
+
# Atomically increment the counter and detect if this update caused the
|
|
13
|
+
# batch to finish. Uses row-level locking to prevent duplicate callbacks.
|
|
14
|
+
# Returns { just_finished:, record: } or nil if batch not found.
|
|
15
|
+
def self.increment_counter!(batch_id, column)
|
|
16
|
+
raise ArgumentError, "Invalid column: #{column}" unless COUNTER_COLUMNS.include?(column)
|
|
17
|
+
|
|
18
|
+
transaction do
|
|
19
|
+
record = lock.find_by(batch_id: batch_id)
|
|
20
|
+
return nil unless record
|
|
21
|
+
|
|
22
|
+
record.increment!(column)
|
|
23
|
+
|
|
24
|
+
just_finished = record.completed_jobs + record.discarded_jobs == record.total_jobs
|
|
25
|
+
record.update!(status: "finished", finished_at: Time.current) if just_finished && record.status != "finished"
|
|
26
|
+
|
|
27
|
+
{ record: record, just_finished: just_finished && record.status == "finished" }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class BlockedExecution < ApplicationRecord
|
|
5
|
+
self.table_name = "pgbus_blocked_executions"
|
|
6
|
+
|
|
7
|
+
scope :for_key, ->(key) { where(concurrency_key: key) }
|
|
8
|
+
scope :expired, ->(now = Time.current) { where("expires_at < ?", now) }
|
|
9
|
+
|
|
10
|
+
# Atomic dequeue: DELETE the highest-priority non-expired row with FOR UPDATE SKIP LOCKED.
|
|
11
|
+
# Returns { queue_name:, payload: } or nil.
|
|
12
|
+
def self.release_next!(concurrency_key)
|
|
13
|
+
now = Time.now.utc
|
|
14
|
+
result = connection.exec_query(
|
|
15
|
+
<<~SQL,
|
|
16
|
+
DELETE FROM pgbus_blocked_executions
|
|
17
|
+
WHERE id = (
|
|
18
|
+
SELECT id FROM pgbus_blocked_executions
|
|
19
|
+
WHERE concurrency_key = $1
|
|
20
|
+
AND expires_at >= $2
|
|
21
|
+
ORDER BY priority ASC, created_at ASC
|
|
22
|
+
LIMIT 1
|
|
23
|
+
FOR UPDATE SKIP LOCKED
|
|
24
|
+
)
|
|
25
|
+
RETURNING queue_name, payload
|
|
26
|
+
SQL
|
|
27
|
+
"Pgbus Blocked Release",
|
|
28
|
+
[concurrency_key, now]
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
row = result.first
|
|
32
|
+
return nil unless row
|
|
33
|
+
|
|
34
|
+
payload = row["payload"]
|
|
35
|
+
payload = JSON.parse(payload) if payload.is_a?(String)
|
|
36
|
+
|
|
37
|
+
{ queue_name: row["queue_name"], payload: payload }
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class RecurringExecution < ApplicationRecord
|
|
5
|
+
self.table_name = "pgbus_recurring_executions"
|
|
6
|
+
|
|
7
|
+
validates :task_key, presence: true
|
|
8
|
+
validates :run_at, presence: true
|
|
9
|
+
|
|
10
|
+
scope :for_task, ->(key) { where(task_key: key) }
|
|
11
|
+
scope :recent, ->(limit = 50) { order(run_at: :desc).limit(limit) }
|
|
12
|
+
scope :older_than, ->(time) { where("run_at < ?", time) }
|
|
13
|
+
|
|
14
|
+
# Record a recurring execution with deduplication.
|
|
15
|
+
# Uses the unique index on (task_key, run_at) to prevent duplicates.
|
|
16
|
+
# Yields to the caller to perform the actual enqueue.
|
|
17
|
+
def self.record(task_key, run_at)
|
|
18
|
+
transaction do
|
|
19
|
+
execution = create!(task_key: task_key, run_at: run_at)
|
|
20
|
+
yield execution if block_given?
|
|
21
|
+
execution
|
|
22
|
+
end
|
|
23
|
+
rescue ActiveRecord::RecordNotUnique
|
|
24
|
+
raise Pgbus::Recurring::AlreadyRecorded,
|
|
25
|
+
"Recurring task '#{task_key}' already recorded for #{run_at.iso8601}"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Find the most recent execution for a task
|
|
29
|
+
def self.last_execution(task_key)
|
|
30
|
+
for_task(task_key).order(run_at: :desc).first
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class RecurringTask < ApplicationRecord
|
|
5
|
+
self.table_name = "pgbus_recurring_tasks"
|
|
6
|
+
|
|
7
|
+
validates :key, presence: true, uniqueness: true
|
|
8
|
+
validates :schedule, presence: true
|
|
9
|
+
|
|
10
|
+
scope :static_tasks, -> { where(static: true) }
|
|
11
|
+
scope :enabled, -> { where(enabled: true) }
|
|
12
|
+
scope :disabled, -> { where(enabled: false) }
|
|
13
|
+
|
|
14
|
+
# Sync static tasks from configuration.
|
|
15
|
+
# Creates new tasks, updates existing ones, removes stale ones.
|
|
16
|
+
def self.sync_from_config!(tasks_hash)
|
|
17
|
+
transaction do
|
|
18
|
+
task_keys = tasks_hash.keys
|
|
19
|
+
|
|
20
|
+
# Upsert all configured tasks
|
|
21
|
+
tasks_hash.each do |key, options|
|
|
22
|
+
options = options.transform_keys(&:to_s)
|
|
23
|
+
record = find_or_initialize_by(key: key)
|
|
24
|
+
record.assign_attributes(
|
|
25
|
+
class_name: options["class"],
|
|
26
|
+
command: options["command"],
|
|
27
|
+
schedule: options["schedule"],
|
|
28
|
+
queue_name: options["queue"],
|
|
29
|
+
arguments: options["args"],
|
|
30
|
+
priority: options.fetch("priority", 0).to_i,
|
|
31
|
+
description: options["description"],
|
|
32
|
+
static: true
|
|
33
|
+
)
|
|
34
|
+
record.save!
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Remove static tasks no longer in config
|
|
38
|
+
static_tasks.where.not(key: task_keys).delete_all
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Semaphore < ApplicationRecord
|
|
5
|
+
self.table_name = "pgbus_semaphores"
|
|
6
|
+
|
|
7
|
+
scope :expired, ->(now = Time.current) { where("expires_at < ? OR value <= 0", now) }
|
|
8
|
+
|
|
9
|
+
# Atomic conditional UPSERT. Returns :acquired or :blocked.
|
|
10
|
+
def self.acquire!(key, max_value, expires_at)
|
|
11
|
+
result = connection.exec_query(
|
|
12
|
+
<<~SQL,
|
|
13
|
+
INSERT INTO pgbus_semaphores (key, value, max_value, expires_at)
|
|
14
|
+
VALUES ($1, 1, $2, $3)
|
|
15
|
+
ON CONFLICT (key) DO UPDATE
|
|
16
|
+
SET value = pgbus_semaphores.value + 1,
|
|
17
|
+
max_value = EXCLUDED.max_value,
|
|
18
|
+
expires_at = GREATEST(pgbus_semaphores.expires_at, EXCLUDED.expires_at)
|
|
19
|
+
WHERE pgbus_semaphores.value < EXCLUDED.max_value
|
|
20
|
+
RETURNING value
|
|
21
|
+
SQL
|
|
22
|
+
"Pgbus Semaphore Acquire",
|
|
23
|
+
[key, max_value, expires_at]
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
result.any? ? :acquired : :blocked
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
<%= pgbus_nav_link "Dashboard", pgbus.root_path %>
|
|
43
43
|
<%= pgbus_nav_link "Queues", pgbus.queues_path %>
|
|
44
44
|
<%= pgbus_nav_link "Jobs", pgbus.jobs_path %>
|
|
45
|
+
<%= pgbus_nav_link "Recurring", pgbus.recurring_tasks_path %>
|
|
45
46
|
<%= pgbus_nav_link "Processes", pgbus.processes_path %>
|
|
46
47
|
<%= pgbus_nav_link "Events", pgbus.events_path %>
|
|
47
48
|
<%= pgbus_nav_link "DLQ", pgbus.dead_letter_index_path %>
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<turbo-frame id="dashboard-stats" data-auto-refresh src="<%= pgbus.root_path(frame: 'stats') %>">
|
|
2
|
-
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-
|
|
2
|
+
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-5 mb-8">
|
|
3
3
|
<div class="rounded-lg bg-white p-5 shadow ring-1 ring-gray-200">
|
|
4
4
|
<p class="text-sm font-medium text-gray-500">Queues</p>
|
|
5
5
|
<p class="mt-1 text-3xl font-semibold text-gray-900"><%= @stats[:total_queues] %></p>
|
|
@@ -18,6 +18,14 @@
|
|
|
18
18
|
</p>
|
|
19
19
|
</div>
|
|
20
20
|
|
|
21
|
+
<div class="rounded-lg bg-white p-5 shadow ring-1 ring-gray-200">
|
|
22
|
+
<p class="text-sm font-medium text-gray-500">Recurring</p>
|
|
23
|
+
<p class="mt-1 text-3xl font-semibold text-gray-900"><%= @stats[:recurring_count] %></p>
|
|
24
|
+
<p class="text-xs text-gray-400">
|
|
25
|
+
<%= link_to "View tasks", pgbus.recurring_tasks_path, class: "text-blue-500 hover:text-blue-700" %>
|
|
26
|
+
</p>
|
|
27
|
+
</div>
|
|
28
|
+
|
|
21
29
|
<div class="rounded-lg bg-white p-5 shadow ring-1 ring-gray-200">
|
|
22
30
|
<p class="text-sm font-medium text-gray-500">Failed / DLQ</p>
|
|
23
31
|
<p class="mt-1 text-3xl font-semibold <%= (@stats[:failed_count] + @stats[:dlq_depth]) > 0 ? 'text-red-600' : 'text-gray-900' %>">
|
|
@@ -4,35 +4,72 @@
|
|
|
4
4
|
<thead class="bg-gray-50">
|
|
5
5
|
<tr>
|
|
6
6
|
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500">ID</th>
|
|
7
|
+
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500">Job Class</th>
|
|
7
8
|
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500">Source Queue</th>
|
|
8
9
|
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500">Enqueued</th>
|
|
9
10
|
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500">Reads</th>
|
|
10
|
-
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500">Payload</th>
|
|
11
|
-
<th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500">Actions</th>
|
|
12
11
|
</tr>
|
|
13
12
|
</thead>
|
|
14
13
|
<tbody class="divide-y divide-gray-100">
|
|
15
14
|
<% @messages.each do |m| %>
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
<td class="
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
15
|
+
<% payload = pgbus_parse_message(m[:message]) %>
|
|
16
|
+
<% dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix %>
|
|
17
|
+
<% source_queue = m[:queue_name].to_s.delete_suffix(dlq_suffix) %>
|
|
18
|
+
<tr>
|
|
19
|
+
<td colspan="5" class="p-0">
|
|
20
|
+
<details class="group">
|
|
21
|
+
<summary class="flex cursor-pointer hover:bg-gray-50 list-none">
|
|
22
|
+
<span class="w-16 shrink-0 px-4 py-3 text-sm font-mono text-gray-900"><%= m[:msg_id] %></span>
|
|
23
|
+
<span class="flex-1 px-4 py-3 text-sm font-medium text-gray-900"><%= payload["job_class"] || "—" %></span>
|
|
24
|
+
<span class="w-40 shrink-0 px-4 py-3 text-sm text-gray-700"><%= source_queue %></span>
|
|
25
|
+
<span class="w-28 shrink-0 px-4 py-3 text-sm text-gray-500"><%= pgbus_time_ago(m[:enqueued_at]) %></span>
|
|
26
|
+
<span class="w-16 shrink-0 px-4 py-3 text-sm text-gray-500"><%= m[:read_ct] %></span>
|
|
27
|
+
</summary>
|
|
28
|
+
<div class="px-4 pb-4 bg-gray-50 border-t border-gray-100">
|
|
29
|
+
<div class="flex items-center justify-between mt-3 mb-3">
|
|
30
|
+
<span class="text-xs font-mono text-gray-400">Job ID: <%= payload["job_id"] %></span>
|
|
31
|
+
<div class="flex space-x-2">
|
|
32
|
+
<%= button_to "Retry", pgbus.retry_dead_letter_path(m[:msg_id], queue_name: m[:queue_name]), method: :post,
|
|
33
|
+
class: "rounded-md bg-indigo-600 px-3 py-1.5 text-xs font-medium text-white hover:bg-indigo-500",
|
|
34
|
+
data: { turbo_frame: "_top" } %>
|
|
35
|
+
<%= button_to "Discard", pgbus.discard_dead_letter_path(m[:msg_id], queue_name: m[:queue_name]), method: :post,
|
|
36
|
+
class: "rounded-md bg-red-600 px-3 py-1.5 text-xs font-medium text-white hover:bg-red-500",
|
|
37
|
+
data: { turbo_confirm: "Permanently discard?", turbo_frame: "_top" } %>
|
|
38
|
+
</div>
|
|
39
|
+
</div>
|
|
40
|
+
<div class="grid grid-cols-2 gap-4 mb-3">
|
|
41
|
+
<div>
|
|
42
|
+
<span class="text-xs font-medium text-gray-500">Arguments</span>
|
|
43
|
+
<pre class="text-xs text-gray-700 bg-white rounded p-2 mt-1 overflow-x-auto max-h-40"><%= JSON.pretty_generate(payload["arguments"] || []) rescue "—" %></pre>
|
|
44
|
+
</div>
|
|
45
|
+
<div>
|
|
46
|
+
<span class="text-xs font-medium text-gray-500">Metadata</span>
|
|
47
|
+
<div class="text-xs text-gray-600 bg-white rounded p-2 mt-1 space-y-1">
|
|
48
|
+
<% if payload["queue_name"] %><p><strong>Queue:</strong> <%= payload["queue_name"] %></p><% end %>
|
|
49
|
+
<% if payload["priority"] %><p><strong>Priority:</strong> <%= payload["priority"] %></p><% end %>
|
|
50
|
+
<% if payload["executions"] %><p><strong>Executions:</strong> <%= payload["executions"] %></p><% end %>
|
|
51
|
+
<% if m[:vt] %><p><strong>Visible at:</strong> <%= m[:vt] %></p><% end %>
|
|
52
|
+
<% if m[:last_read_at] %><p><strong>Last read:</strong> <%= m[:last_read_at] %></p><% end %>
|
|
53
|
+
</div>
|
|
54
|
+
</div>
|
|
55
|
+
</div>
|
|
56
|
+
<details class="mt-2">
|
|
57
|
+
<summary class="text-xs font-medium text-gray-500 cursor-pointer hover:text-gray-700">Full JSON payload</summary>
|
|
58
|
+
<pre class="text-xs text-gray-600 bg-white rounded p-2 mt-1 overflow-x-auto max-h-96"><%= JSON.pretty_generate(payload) rescue m[:message] %></pre>
|
|
59
|
+
</details>
|
|
60
|
+
<% if m[:headers] %>
|
|
61
|
+
<details class="mt-2">
|
|
62
|
+
<summary class="text-xs font-medium text-gray-500 cursor-pointer hover:text-gray-700">Headers</summary>
|
|
63
|
+
<pre class="text-xs text-gray-600 bg-white rounded p-2 mt-1 overflow-x-auto"><%= JSON.pretty_generate(JSON.parse(m[:headers])) rescue m[:headers] %></pre>
|
|
64
|
+
</details>
|
|
65
|
+
<% end %>
|
|
66
|
+
</div>
|
|
67
|
+
</details>
|
|
31
68
|
</td>
|
|
32
69
|
</tr>
|
|
33
70
|
<% end %>
|
|
34
71
|
<% if @messages.empty? %>
|
|
35
|
-
<tr><td colspan="
|
|
72
|
+
<tr><td colspan="5" class="px-4 py-8 text-center text-sm text-gray-400">Dead letter queue is empty</td></tr>
|
|
36
73
|
<% end %>
|
|
37
74
|
</tbody>
|
|
38
75
|
</table>
|