message_bus 4.1.0 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 11a14ba0ec9ad142d3335fb7eb764be8e9b975a8dc8367fd4bf359e128a56095
4
- data.tar.gz: bf41cede85c8e75e56829447cd3cf9ed6cd531ba27fcddfbccb6043f3a82da34
3
+ metadata.gz: edad2fa285f65718a34893791166991c2cdf893b6e7064c5163882cd92f5fc95
4
+ data.tar.gz: 686c4fcc4c02add5adf75fa08586491af36173d67bfac917093b291d8d3343f2
5
5
  SHA512:
6
- metadata.gz: 23c1d181028194c16e98c5a3df42e445458aef4779c97ae2d6d5100adf5053680f62ea4c914bea7c84c124bd453127e141c68ffeb1c742cc208e1d040bc8558f
7
- data.tar.gz: 7efffe711fac2520356b115820acba2d4206d53f35017ba64c86e3c571a85fb423e943cb9301a03d3b47d6f442f1626dea0495c0242220283685eaa944c890b6
6
+ metadata.gz: fb23eadc6112f7d28128b46e1e00d7bbdd196cf574398a3746927210d9affd4f127b83e6a9f104a65d8909d03729af16221ef7ec159126cdc5208d6eacf3e2d5
7
+ data.tar.gz: bdbf6a4d6d3c129f58a8c0fb02d5275cda59becb59120a602423340597e7afcb84410b39f58b537a2962a0157d0a1ca10d2efbc28f6bfbea4e3fe160a7cf46f7
data/CHANGELOG CHANGED
@@ -1,3 +1,11 @@
1
+ 22-02-2022
2
+
3
+ - Version 4.2.0
4
+
5
+ - FEATURE: Add support for `clear_every` parameter in Redis backend
6
+
7
+ This allows the clearing of the backlog to take place once every N messages. In high-load scenarios, this can provide significant performance benefit.
8
+
1
9
  16-02-2022
2
10
 
3
11
  - Version 4.1.0
data/README.md CHANGED
@@ -388,7 +388,9 @@ The redis client message_bus uses is [redis-rb](https://github.com/redis/redis-r
388
388
 
389
389
  #### Data Retention
390
390
 
391
- Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get cleared automatically after 7 days of inactivity.
391
+ Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get
392
+ cleared automatically after 7 days of inactivity. By default, the backlog will be pruned on every message publication. If exact backlog
393
+ length limiting is not required, the `clear_every` parameter can be set higher to improve performance.
392
394
 
393
395
  This is configurable via accessors on the Backend instance.
394
396
 
@@ -401,6 +403,9 @@ MessageBus.backend_instance.max_global_backlog_size = 100
401
403
 
402
404
  # flush per-channel backlog after 100 seconds of inactivity
403
405
  MessageBus.backend_instance.max_backlog_age = 100
406
+
407
+ # clear the backlog every 50 messages
408
+ MessageBus.backend_instance.clear_every = 50
404
409
  ```
405
410
 
406
411
  ### PostgreSQL
@@ -423,7 +428,7 @@ message_bus also supports an in-memory backend. This can be used for testing or
423
428
  MessageBus.configure(backend: :memory)
424
429
  ```
425
430
 
426
- The `:clear_every` option supported by the PostgreSQL backend is also supported by the in-memory backend.
431
+ The `:clear_every` option is supported in the same way as the PostgreSQL backend.
427
432
 
428
433
  ### Transport codecs
429
434
 
@@ -30,8 +30,6 @@ module MessageBus
30
30
  # messages are removed (when no publication happens during this
31
31
  # time-frame).
32
32
  #
33
- # * `clear_every` is not a supported option for this backend.
34
- #
35
33
  # @see Base general information about message_bus backends
36
34
  class Redis < Base
37
35
  class BackLogOutOfOrder < StandardError
@@ -45,9 +43,11 @@ module MessageBus
45
43
  # @param [Hash] redis_config in addition to the options listed, see https://github.com/redis/redis-rb for other available options
46
44
  # @option redis_config [Logger] :logger a logger to which logs will be output
47
45
  # @option redis_config [Boolean] :enable_redis_logger (false) whether or not to enable logging by the underlying Redis library
46
+ # @option redis_config [Integer] :clear_every (1) the interval of publications between which the backlog will not be cleared
48
47
  # @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
49
48
  def initialize(redis_config = {}, max_backlog_size = 1000)
50
49
  @redis_config = redis_config.dup
50
+ @clear_every = redis_config.delete(:clear_every) || 1
51
51
  @logger = @redis_config[:logger]
52
52
  unless @redis_config[:enable_redis_logger]
53
53
  @redis_config[:logger] = nil
@@ -100,6 +100,7 @@ module MessageBus
100
100
  local max_backlog_size = tonumber(ARGV[3])
101
101
  local max_global_backlog_size = tonumber(ARGV[4])
102
102
  local channel = ARGV[5]
103
+ local clear_every = ARGV[6]
103
104
 
104
105
  local global_id_key = KEYS[1]
105
106
  local backlog_id_key = KEYS[2]
@@ -120,11 +121,11 @@ module MessageBus
120
121
 
121
122
  redis.call("EXPIRE", backlog_id_key, max_backlog_age)
122
123
 
123
- if backlog_id > max_backlog_size then
124
+ if backlog_id > max_backlog_size and backlog_id % clear_every == 0 then
124
125
  redis.call("ZREMRANGEBYSCORE", backlog_key, 1, backlog_id - max_backlog_size)
125
126
  end
126
127
 
127
- if global_id > max_global_backlog_size then
128
+ if global_id > max_global_backlog_size and global_id % clear_every == 0 then
128
129
  redis.call("ZREMRANGEBYSCORE", global_backlog_key, 1, global_id - max_global_backlog_size)
129
130
  end
130
131
 
@@ -155,7 +156,8 @@ LUA
155
156
  max_backlog_age,
156
157
  max_backlog_size,
157
158
  max_global_backlog_size,
158
- channel
159
+ channel,
160
+ clear_every
159
161
  ],
160
162
  keys: [
161
163
  global_id_key,
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module MessageBus
4
- VERSION = "4.1.0"
4
+ VERSION = "4.2.0"
5
5
  end
@@ -310,8 +310,6 @@ describe BACKEND_CLASS do
310
310
  end
311
311
 
312
312
  it "should support clear_every setting" do
313
- test_never :redis
314
-
315
313
  @bus.clear_every = 5
316
314
  @bus.max_global_backlog_size = 2
317
315
  @bus.publish "/foo", "11"
@@ -9,7 +9,7 @@ require_relative "../helpers"
9
9
 
10
10
  backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym)
11
11
  channel = "/foo"
12
- iterations = 10_000
12
+ iterations = 100_000
13
13
  results = []
14
14
 
15
15
  puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}"
@@ -78,6 +78,33 @@ benchmark_subscription_with_trimming = lambda do |bm, backend|
78
78
  bus.destroy
79
79
  end
80
80
 
81
+ benchmark_subscription_with_trimming_and_clear_every = lambda do |bm, backend|
82
+ test_title = "#{backend} - subscription with trimming and clear_every=50"
83
+
84
+ bus = MessageBus::Instance.new
85
+ bus.configure(test_config_for_backend(backend))
86
+
87
+ bus.backend_instance.max_backlog_size = (iterations / 10)
88
+ bus.backend_instance.max_global_backlog_size = (iterations / 10)
89
+ bus.backend_instance.clear_every = 50
90
+
91
+ messages_received = 0
92
+ bus.after_fork
93
+ bus.subscribe(channel) do |_message|
94
+ messages_received += 1
95
+ end
96
+
97
+ bm.report(test_title) do
98
+ iterations.times { bus.publish(channel, "Hello world") }
99
+ wait_for(60000) { messages_received == iterations }
100
+ end
101
+
102
+ results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%"
103
+
104
+ bus.reset!
105
+ bus.destroy
106
+ end
107
+
81
108
  puts
82
109
  Benchmark.bm(60) do |bm|
83
110
  backends.each do |backend|
@@ -96,6 +123,10 @@ Benchmark.bm(60) do |bm|
96
123
  backends.each do |backend|
97
124
  benchmark_subscription_with_trimming.call(bm, backend)
98
125
  end
126
+
127
+ backends.each do |backend|
128
+ benchmark_subscription_with_trimming_and_clear_every.call(bm, backend)
129
+ end
99
130
  end
100
131
  puts
101
132
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: message_bus
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.1.0
4
+ version: 4.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam Saffron
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-02-17 00:00:00.000000000 Z
11
+ date: 2022-02-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rack