message_bus 4.1.0 → 4.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 11a14ba0ec9ad142d3335fb7eb764be8e9b975a8dc8367fd4bf359e128a56095
4
- data.tar.gz: bf41cede85c8e75e56829447cd3cf9ed6cd531ba27fcddfbccb6043f3a82da34
3
+ metadata.gz: edad2fa285f65718a34893791166991c2cdf893b6e7064c5163882cd92f5fc95
4
+ data.tar.gz: 686c4fcc4c02add5adf75fa08586491af36173d67bfac917093b291d8d3343f2
5
5
  SHA512:
6
- metadata.gz: 23c1d181028194c16e98c5a3df42e445458aef4779c97ae2d6d5100adf5053680f62ea4c914bea7c84c124bd453127e141c68ffeb1c742cc208e1d040bc8558f
7
- data.tar.gz: 7efffe711fac2520356b115820acba2d4206d53f35017ba64c86e3c571a85fb423e943cb9301a03d3b47d6f442f1626dea0495c0242220283685eaa944c890b6
6
+ metadata.gz: fb23eadc6112f7d28128b46e1e00d7bbdd196cf574398a3746927210d9affd4f127b83e6a9f104a65d8909d03729af16221ef7ec159126cdc5208d6eacf3e2d5
7
+ data.tar.gz: bdbf6a4d6d3c129f58a8c0fb02d5275cda59becb59120a602423340597e7afcb84410b39f58b537a2962a0157d0a1ca10d2efbc28f6bfbea4e3fe160a7cf46f7
data/CHANGELOG CHANGED
@@ -1,3 +1,11 @@
1
+ 22-02-2022
2
+
3
+ - Version 4.2.0
4
+
5
+ - FEATURE: Add support for `clear_every` parameter in Redis backend
6
+
7
+ This allows the clearing of the backlog to take place once every N messages. In high-load scenarios, this can provide significant performance benefit.
8
+
1
9
  16-02-2022
2
10
 
3
11
  - Version 4.1.0
data/README.md CHANGED
@@ -388,7 +388,9 @@ The redis client message_bus uses is [redis-rb](https://github.com/redis/redis-r
388
388
 
389
389
  #### Data Retention
390
390
 
391
- Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get cleared automatically after 7 days of inactivity.
391
+ Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get
392
+ cleared automatically after 7 days of inactivity. By default, the backlog will be pruned on every message publication. If exact backlog
393
+ length limiting is not required, the `clear_every` parameter can be set higher to improve performance.
392
394
 
393
395
  This is configurable via accessors on the Backend instance.
394
396
 
@@ -401,6 +403,9 @@ MessageBus.backend_instance.max_global_backlog_size = 100
401
403
 
402
404
  # flush per-channel backlog after 100 seconds of inactivity
403
405
  MessageBus.backend_instance.max_backlog_age = 100
406
+
407
+ # clear the backlog every 50 messages
408
+ MessageBus.backend_instance.clear_every = 50
404
409
  ```
405
410
 
406
411
  ### PostgreSQL
@@ -423,7 +428,7 @@ message_bus also supports an in-memory backend. This can be used for testing or
423
428
  MessageBus.configure(backend: :memory)
424
429
  ```
425
430
 
426
- The `:clear_every` option supported by the PostgreSQL backend is also supported by the in-memory backend.
431
+ The `:clear_every` option is supported in the same way as the PostgreSQL backend.
427
432
 
428
433
  ### Transport codecs
429
434
 
@@ -30,8 +30,6 @@ module MessageBus
30
30
  # messages are removed (when no publication happens during this
31
31
  # time-frame).
32
32
  #
33
- # * `clear_every` is not a supported option for this backend.
34
- #
35
33
  # @see Base general information about message_bus backends
36
34
  class Redis < Base
37
35
  class BackLogOutOfOrder < StandardError
@@ -45,9 +43,11 @@ module MessageBus
45
43
  # @param [Hash] redis_config in addition to the options listed, see https://github.com/redis/redis-rb for other available options
46
44
  # @option redis_config [Logger] :logger a logger to which logs will be output
47
45
  # @option redis_config [Boolean] :enable_redis_logger (false) whether or not to enable logging by the underlying Redis library
46
+ # @option redis_config [Integer] :clear_every (1) the interval of publications between which the backlog will not be cleared
48
47
  # @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
49
48
  def initialize(redis_config = {}, max_backlog_size = 1000)
50
49
  @redis_config = redis_config.dup
50
+ @clear_every = redis_config.delete(:clear_every) || 1
51
51
  @logger = @redis_config[:logger]
52
52
  unless @redis_config[:enable_redis_logger]
53
53
  @redis_config[:logger] = nil
@@ -100,6 +100,7 @@ module MessageBus
100
100
  local max_backlog_size = tonumber(ARGV[3])
101
101
  local max_global_backlog_size = tonumber(ARGV[4])
102
102
  local channel = ARGV[5]
103
+ local clear_every = ARGV[6]
103
104
 
104
105
  local global_id_key = KEYS[1]
105
106
  local backlog_id_key = KEYS[2]
@@ -120,11 +121,11 @@ module MessageBus
120
121
 
121
122
  redis.call("EXPIRE", backlog_id_key, max_backlog_age)
122
123
 
123
- if backlog_id > max_backlog_size then
124
+ if backlog_id > max_backlog_size and backlog_id % clear_every == 0 then
124
125
  redis.call("ZREMRANGEBYSCORE", backlog_key, 1, backlog_id - max_backlog_size)
125
126
  end
126
127
 
127
- if global_id > max_global_backlog_size then
128
+ if global_id > max_global_backlog_size and global_id % clear_every == 0 then
128
129
  redis.call("ZREMRANGEBYSCORE", global_backlog_key, 1, global_id - max_global_backlog_size)
129
130
  end
130
131
 
@@ -155,7 +156,8 @@ LUA
155
156
  max_backlog_age,
156
157
  max_backlog_size,
157
158
  max_global_backlog_size,
158
- channel
159
+ channel,
160
+ clear_every
159
161
  ],
160
162
  keys: [
161
163
  global_id_key,
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module MessageBus
4
- VERSION = "4.1.0"
4
+ VERSION = "4.2.0"
5
5
  end
@@ -310,8 +310,6 @@ describe BACKEND_CLASS do
310
310
  end
311
311
 
312
312
  it "should support clear_every setting" do
313
- test_never :redis
314
-
315
313
  @bus.clear_every = 5
316
314
  @bus.max_global_backlog_size = 2
317
315
  @bus.publish "/foo", "11"
@@ -9,7 +9,7 @@ require_relative "../helpers"
9
9
 
10
10
  backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym)
11
11
  channel = "/foo"
12
- iterations = 10_000
12
+ iterations = 100_000
13
13
  results = []
14
14
 
15
15
  puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}"
@@ -78,6 +78,33 @@ benchmark_subscription_with_trimming = lambda do |bm, backend|
78
78
  bus.destroy
79
79
  end
80
80
 
81
+ benchmark_subscription_with_trimming_and_clear_every = lambda do |bm, backend|
82
+ test_title = "#{backend} - subscription with trimming and clear_every=50"
83
+
84
+ bus = MessageBus::Instance.new
85
+ bus.configure(test_config_for_backend(backend))
86
+
87
+ bus.backend_instance.max_backlog_size = (iterations / 10)
88
+ bus.backend_instance.max_global_backlog_size = (iterations / 10)
89
+ bus.backend_instance.clear_every = 50
90
+
91
+ messages_received = 0
92
+ bus.after_fork
93
+ bus.subscribe(channel) do |_message|
94
+ messages_received += 1
95
+ end
96
+
97
+ bm.report(test_title) do
98
+ iterations.times { bus.publish(channel, "Hello world") }
99
+ wait_for(60000) { messages_received == iterations }
100
+ end
101
+
102
+ results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%"
103
+
104
+ bus.reset!
105
+ bus.destroy
106
+ end
107
+
81
108
  puts
82
109
  Benchmark.bm(60) do |bm|
83
110
  backends.each do |backend|
@@ -96,6 +123,10 @@ Benchmark.bm(60) do |bm|
96
123
  backends.each do |backend|
97
124
  benchmark_subscription_with_trimming.call(bm, backend)
98
125
  end
126
+
127
+ backends.each do |backend|
128
+ benchmark_subscription_with_trimming_and_clear_every.call(bm, backend)
129
+ end
99
130
  end
100
131
  puts
101
132
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: message_bus
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.1.0
4
+ version: 4.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam Saffron
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-02-17 00:00:00.000000000 Z
11
+ date: 2022-02-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rack