message_bus 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11a14ba0ec9ad142d3335fb7eb764be8e9b975a8dc8367fd4bf359e128a56095
-  data.tar.gz: bf41cede85c8e75e56829447cd3cf9ed6cd531ba27fcddfbccb6043f3a82da34
+  metadata.gz: edad2fa285f65718a34893791166991c2cdf893b6e7064c5163882cd92f5fc95
+  data.tar.gz: 686c4fcc4c02add5adf75fa08586491af36173d67bfac917093b291d8d3343f2
 SHA512:
-  metadata.gz: 23c1d181028194c16e98c5a3df42e445458aef4779c97ae2d6d5100adf5053680f62ea4c914bea7c84c124bd453127e141c68ffeb1c742cc208e1d040bc8558f
-  data.tar.gz: 7efffe711fac2520356b115820acba2d4206d53f35017ba64c86e3c571a85fb423e943cb9301a03d3b47d6f442f1626dea0495c0242220283685eaa944c890b6
+  metadata.gz: fb23eadc6112f7d28128b46e1e00d7bbdd196cf574398a3746927210d9affd4f127b83e6a9f104a65d8909d03729af16221ef7ec159126cdc5208d6eacf3e2d5
+  data.tar.gz: bdbf6a4d6d3c129f58a8c0fb02d5275cda59becb59120a602423340597e7afcb84410b39f58b537a2962a0157d0a1ca10d2efbc28f6bfbea4e3fe160a7cf46f7

data/CHANGELOG

@@ -1,3 +1,11 @@
+22-02-2022
+
+- Version 4.2.0
+
+  - FEATURE: Add support for `clear_every` parameter in Redis backend
+
+    This allows the clearing of the backlog to take place once every N messages. In high-load scenarios, this can provide significant performance benefit.
+
 16-02-2022
 
 - Version 4.1.0

data/README.md

@@ -388,7 +388,9 @@ The redis client message_bus uses is [redis-rb](https://github.com/redis/redis-r
 
 #### Data Retention
 
-Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get cleared automatically after 7 days of inactivity.
+Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get
+cleared automatically after 7 days of inactivity. By default, the backlog will be pruned on every message publication. If exact backlog
+length limiting is not required, the `clear_every` parameter can be set higher to improve performance.
 
 This is configurable via accessors on the Backend instance.
 
@@ -401,6 +403,9 @@ MessageBus.backend_instance.max_global_backlog_size = 100
 
 # flush per-channel backlog after 100 seconds of inactivity
 MessageBus.backend_instance.max_backlog_age = 100
+
+# clear the backlog every 50 messages
+MessageBus.backend_instance.clear_every = 50
 ```
 
 ### PostgreSQL
@@ -423,7 +428,7 @@ message_bus also supports an in-memory backend. This can be used for testing or
 MessageBus.configure(backend: :memory)
 ```
 
-The `:clear_every` option supported by the PostgreSQL backend is also supported by the in-memory backend.
+The `:clear_every` option is supported in the same way as the PostgreSQL backend.
 
 ### Transport codecs
 

data/lib/message_bus/backends/redis.rb

@@ -30,8 +30,6 @@ module MessageBus
     #     messages are removed (when no publication happens during this
     #     time-frame).
     #
-    #   * `clear_every` is not a supported option for this backend.
-    #
     # @see Base general information about message_bus backends
     class Redis < Base
       class BackLogOutOfOrder < StandardError
@@ -45,9 +43,11 @@ module MessageBus
       # @param [Hash] redis_config in addition to the options listed, see https://github.com/redis/redis-rb for other available options
       # @option redis_config [Logger] :logger a logger to which logs will be output
       # @option redis_config [Boolean] :enable_redis_logger (false) whether or not to enable logging by the underlying Redis library
+      # @option redis_config [Integer] :clear_every (1) the interval of publications between which the backlog will not be cleared
       # @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
       def initialize(redis_config = {}, max_backlog_size = 1000)
         @redis_config = redis_config.dup
+        @clear_every = redis_config.delete(:clear_every) || 1
         @logger = @redis_config[:logger]
         unless @redis_config[:enable_redis_logger]
           @redis_config[:logger] = nil
@@ -100,6 +100,7 @@ module MessageBus
       local max_backlog_size = tonumber(ARGV[3])
       local max_global_backlog_size = tonumber(ARGV[4])
       local channel = ARGV[5]
+      local clear_every = ARGV[6]
 
       local global_id_key = KEYS[1]
       local backlog_id_key = KEYS[2]
@@ -120,11 +121,11 @@ module MessageBus
 
       redis.call("EXPIRE", backlog_id_key, max_backlog_age)
 
-      if backlog_id > max_backlog_size then
+      if backlog_id > max_backlog_size and backlog_id % clear_every == 0 then
         redis.call("ZREMRANGEBYSCORE", backlog_key, 1, backlog_id - max_backlog_size)
       end
 
-      if global_id > max_global_backlog_size then
+      if global_id > max_global_backlog_size and global_id % clear_every == 0 then
         redis.call("ZREMRANGEBYSCORE", global_backlog_key, 1, global_id - max_global_backlog_size)
       end
 
@@ -155,7 +156,8 @@ LUA
             max_backlog_age,
             max_backlog_size,
             max_global_backlog_size,
-            channel
+            channel,
+            clear_every
           ],
           keys: [
             global_id_key,

data/lib/message_bus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MessageBus
-  VERSION = "4.1.0"
+  VERSION = "4.2.0"
 end

data/spec/lib/message_bus/backend_spec.rb

@@ -310,8 +310,6 @@ describe BACKEND_CLASS do
   end
 
   it "should support clear_every setting" do
-    test_never :redis
-
     @bus.clear_every = 5
     @bus.max_global_backlog_size = 2
     @bus.publish "/foo", "11"

data/spec/performance/publish.rb

@@ -9,7 +9,7 @@ require_relative "../helpers"
 
 backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym)
 channel = "/foo"
-iterations = 10_000
+iterations = 100_000
 results = []
 
 puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}"
@@ -78,6 +78,33 @@ benchmark_subscription_with_trimming = lambda do |bm, backend|
   bus.destroy
 end
 
+benchmark_subscription_with_trimming_and_clear_every = lambda do |bm, backend|
+  test_title = "#{backend} - subscription with trimming and clear_every=50"
+
+  bus = MessageBus::Instance.new
+  bus.configure(test_config_for_backend(backend))
+
+  bus.backend_instance.max_backlog_size = (iterations / 10)
+  bus.backend_instance.max_global_backlog_size = (iterations / 10)
+  bus.backend_instance.clear_every = 50
+
+  messages_received = 0
+  bus.after_fork
+  bus.subscribe(channel) do |_message|
+    messages_received += 1
+  end
+
+  bm.report(test_title) do
+    iterations.times { bus.publish(channel, "Hello world") }
+    wait_for(60000) { messages_received == iterations }
+  end
+
+  results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%"
+
+  bus.reset!
+  bus.destroy
+end
+
 puts
 Benchmark.bm(60) do |bm|
   backends.each do |backend|
@@ -96,6 +123,10 @@ Benchmark.bm(60) do |bm|
   backends.each do |backend|
     benchmark_subscription_with_trimming.call(bm, backend)
   end
+
+  backends.each do |backend|
+    benchmark_subscription_with_trimming_and_clear_every.call(bm, backend)
+  end
 end
 puts
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: message_bus
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Sam Saffron
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-17 00:00:00.000000000 Z
+date: 2022-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack