message_bus 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG +8 -0
- data/README.md +7 -2
- data/lib/message_bus/backends/redis.rb +7 -5
- data/lib/message_bus/version.rb +1 -1
- data/spec/lib/message_bus/backend_spec.rb +0 -2
- data/spec/performance/publish.rb +32 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edad2fa285f65718a34893791166991c2cdf893b6e7064c5163882cd92f5fc95
|
4
|
+
data.tar.gz: 686c4fcc4c02add5adf75fa08586491af36173d67bfac917093b291d8d3343f2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fb23eadc6112f7d28128b46e1e00d7bbdd196cf574398a3746927210d9affd4f127b83e6a9f104a65d8909d03729af16221ef7ec159126cdc5208d6eacf3e2d5
|
7
|
+
data.tar.gz: bdbf6a4d6d3c129f58a8c0fb02d5275cda59becb59120a602423340597e7afcb84410b39f58b537a2962a0157d0a1ca10d2efbc28f6bfbea4e3fe160a7cf46f7
|
data/CHANGELOG
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
22-02-2022
|
2
|
+
|
3
|
+
- Version 4.2.0
|
4
|
+
|
5
|
+
- FEATURE: Add support for `clear_every` parameter in Redis backend
|
6
|
+
|
7
|
+
This allows the clearing of the backlog to take place once every N messages. In high-load scenarios, this can provide significant performance benefit.
|
8
|
+
|
1
9
|
16-02-2022
|
2
10
|
|
3
11
|
- Version 4.1.0
|
data/README.md
CHANGED
@@ -388,7 +388,9 @@ The redis client message_bus uses is [redis-rb](https://github.com/redis/redis-r
|
|
388
388
|
|
389
389
|
#### Data Retention
|
390
390
|
|
391
|
-
Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get
|
391
|
+
Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get
|
392
|
+
cleared automatically after 7 days of inactivity. By default, the backlog will be pruned on every message publication. If exact backlog
|
393
|
+
length limiting is not required, the `clear_every` parameter can be set higher to improve performance.
|
392
394
|
|
393
395
|
This is configurable via accessors on the Backend instance.
|
394
396
|
|
@@ -401,6 +403,9 @@ MessageBus.backend_instance.max_global_backlog_size = 100
|
|
401
403
|
|
402
404
|
# flush per-channel backlog after 100 seconds of inactivity
|
403
405
|
MessageBus.backend_instance.max_backlog_age = 100
|
406
|
+
|
407
|
+
# clear the backlog every 50 messages
|
408
|
+
MessageBus.backend_instance.clear_every = 50
|
404
409
|
```
|
405
410
|
|
406
411
|
### PostgreSQL
|
@@ -423,7 +428,7 @@ message_bus also supports an in-memory backend. This can be used for testing or
|
|
423
428
|
MessageBus.configure(backend: :memory)
|
424
429
|
```
|
425
430
|
|
426
|
-
The `:clear_every` option supported
|
431
|
+
The `:clear_every` option is supported in the same way as the PostgreSQL backend.
|
427
432
|
|
428
433
|
### Transport codecs
|
429
434
|
|
@@ -30,8 +30,6 @@ module MessageBus
|
|
30
30
|
# messages are removed (when no publication happens during this
|
31
31
|
# time-frame).
|
32
32
|
#
|
33
|
-
# * `clear_every` is not a supported option for this backend.
|
34
|
-
#
|
35
33
|
# @see Base general information about message_bus backends
|
36
34
|
class Redis < Base
|
37
35
|
class BackLogOutOfOrder < StandardError
|
@@ -45,9 +43,11 @@ module MessageBus
|
|
45
43
|
# @param [Hash] redis_config in addition to the options listed, see https://github.com/redis/redis-rb for other available options
|
46
44
|
# @option redis_config [Logger] :logger a logger to which logs will be output
|
47
45
|
# @option redis_config [Boolean] :enable_redis_logger (false) whether or not to enable logging by the underlying Redis library
|
46
|
+
# @option redis_config [Integer] :clear_every (1) the interval of publications between which the backlog will not be cleared
|
48
47
|
# @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
|
49
48
|
def initialize(redis_config = {}, max_backlog_size = 1000)
|
50
49
|
@redis_config = redis_config.dup
|
50
|
+
@clear_every = redis_config.delete(:clear_every) || 1
|
51
51
|
@logger = @redis_config[:logger]
|
52
52
|
unless @redis_config[:enable_redis_logger]
|
53
53
|
@redis_config[:logger] = nil
|
@@ -100,6 +100,7 @@ module MessageBus
|
|
100
100
|
local max_backlog_size = tonumber(ARGV[3])
|
101
101
|
local max_global_backlog_size = tonumber(ARGV[4])
|
102
102
|
local channel = ARGV[5]
|
103
|
+
local clear_every = ARGV[6]
|
103
104
|
|
104
105
|
local global_id_key = KEYS[1]
|
105
106
|
local backlog_id_key = KEYS[2]
|
@@ -120,11 +121,11 @@ module MessageBus
|
|
120
121
|
|
121
122
|
redis.call("EXPIRE", backlog_id_key, max_backlog_age)
|
122
123
|
|
123
|
-
if backlog_id > max_backlog_size then
|
124
|
+
if backlog_id > max_backlog_size and backlog_id % clear_every == 0 then
|
124
125
|
redis.call("ZREMRANGEBYSCORE", backlog_key, 1, backlog_id - max_backlog_size)
|
125
126
|
end
|
126
127
|
|
127
|
-
if global_id > max_global_backlog_size then
|
128
|
+
if global_id > max_global_backlog_size and global_id % clear_every == 0 then
|
128
129
|
redis.call("ZREMRANGEBYSCORE", global_backlog_key, 1, global_id - max_global_backlog_size)
|
129
130
|
end
|
130
131
|
|
@@ -155,7 +156,8 @@ LUA
|
|
155
156
|
max_backlog_age,
|
156
157
|
max_backlog_size,
|
157
158
|
max_global_backlog_size,
|
158
|
-
channel
|
159
|
+
channel,
|
160
|
+
clear_every
|
159
161
|
],
|
160
162
|
keys: [
|
161
163
|
global_id_key,
|
data/lib/message_bus/version.rb
CHANGED
data/spec/performance/publish.rb
CHANGED
@@ -9,7 +9,7 @@ require_relative "../helpers"
|
|
9
9
|
|
10
10
|
backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym)
|
11
11
|
channel = "/foo"
|
12
|
-
iterations =
|
12
|
+
iterations = 100_000
|
13
13
|
results = []
|
14
14
|
|
15
15
|
puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}"
|
@@ -78,6 +78,33 @@ benchmark_subscription_with_trimming = lambda do |bm, backend|
|
|
78
78
|
bus.destroy
|
79
79
|
end
|
80
80
|
|
81
|
+
benchmark_subscription_with_trimming_and_clear_every = lambda do |bm, backend|
|
82
|
+
test_title = "#{backend} - subscription with trimming and clear_every=50"
|
83
|
+
|
84
|
+
bus = MessageBus::Instance.new
|
85
|
+
bus.configure(test_config_for_backend(backend))
|
86
|
+
|
87
|
+
bus.backend_instance.max_backlog_size = (iterations / 10)
|
88
|
+
bus.backend_instance.max_global_backlog_size = (iterations / 10)
|
89
|
+
bus.backend_instance.clear_every = 50
|
90
|
+
|
91
|
+
messages_received = 0
|
92
|
+
bus.after_fork
|
93
|
+
bus.subscribe(channel) do |_message|
|
94
|
+
messages_received += 1
|
95
|
+
end
|
96
|
+
|
97
|
+
bm.report(test_title) do
|
98
|
+
iterations.times { bus.publish(channel, "Hello world") }
|
99
|
+
wait_for(60000) { messages_received == iterations }
|
100
|
+
end
|
101
|
+
|
102
|
+
results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%"
|
103
|
+
|
104
|
+
bus.reset!
|
105
|
+
bus.destroy
|
106
|
+
end
|
107
|
+
|
81
108
|
puts
|
82
109
|
Benchmark.bm(60) do |bm|
|
83
110
|
backends.each do |backend|
|
@@ -96,6 +123,10 @@ Benchmark.bm(60) do |bm|
|
|
96
123
|
backends.each do |backend|
|
97
124
|
benchmark_subscription_with_trimming.call(bm, backend)
|
98
125
|
end
|
126
|
+
|
127
|
+
backends.each do |backend|
|
128
|
+
benchmark_subscription_with_trimming_and_clear_every.call(bm, backend)
|
129
|
+
end
|
99
130
|
end
|
100
131
|
puts
|
101
132
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: message_bus
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sam Saffron
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-02-
|
11
|
+
date: 2022-02-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rack
|