message_bus 3.3.1 → 3.3.6

data/bench/codecs/all_codecs.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative './packed_string'
+require_relative './string_hack'
+require_relative './marshal'
+
+def all_codecs
+  {
+    json: MessageBus::Codec::Json.new,
+    oj: MessageBus::Codec::Oj.new,
+    marshal: MarshalCodec.new,
+    packed_string_4_bytes: PackedString.new("V"),
+    packed_string_8_bytes: PackedString.new("Q"),
+    string_hack: StringHack.new
+  }
+end
+
+def bench_decode(hash, user_needle)
+  encoded_data = all_codecs.map do |name, codec|
+    [
+      name, codec, codec.encode(hash.dup)
+    ]
+  end
+
+  Benchmark.ips do |x|
+
+    encoded_data.each do |name, codec, encoded|
+      x.report(name) do |n|
+        while n > 0
+          decoded = codec.decode(encoded)
+          decoded["user_ids"].include?(user_needle)
+          n -= 1
+        end
+      end
+    end
+
+    x.compare!
+  end
+end

data/bench/codecs/marshal.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class MarshalCodec
+  def encode(hash)
+    ::Marshal.dump(hash)
+  end
+
+  def decode(payload)
+    ::Marshal.load(payload)
+  end
+end

data/bench/codecs/packed_string.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class PackedString
+  class FastIdList
+    def self.from_array(array, pack_with)
+      new(array.sort.pack("#{pack_with}*"), pack_with)
+    end
+
+    def self.from_string(string, pack_with)
+      new(string, pack_with)
+    end
+
+    def initialize(packed, pack_with)
+      raise "unknown pack format, expecting Q or V" if pack_with != "V" && pack_with != "Q"
+      @packed = packed
+      @pack_with = pack_with
+      @slot_size = pack_with == "V" ? 4 : 8
+    end
+
+    def include?(id)
+      found = (0...length).bsearch do |index|
+        @packed.byteslice(index * @slot_size, @slot_size).unpack1(@pack_with) >= id
+      end
+
+      found && @packed.byteslice(found * @slot_size, @slot_size).unpack1(@pack_with) == id
+    end
+
+    def length
+      @length ||= @packed.bytesize / @slot_size
+    end
+
+    def to_a
+      @packed.unpack("#{@pack_with}*")
+    end
+
+    def to_s
+      @packed
+    end
+  end
+
+  def initialize(pack_with = "V")
+    @pack_with = pack_with
+    @oj_options = { mode: :compat }
+  end
+
+  def encode(hash)
+
+    if user_ids = hash["user_ids"]
+      hash["user_ids"] = FastIdList.from_array(hash["user_ids"], @pack_with).to_s
+    end
+
+    hash["data"] = ::Oj.dump(hash["data"], @oj_options)
+
+    Marshal.dump(hash)
+  end
+
+  def decode(payload)
+    result = Marshal.load(payload)
+    result["data"] = ::Oj.load(result["data"], @oj_options)
+
+    if str = result["user_ids"]
+      result["user_ids"] = FastIdList.from_string(str, @pack_with)
+    end
+
+    result
+  end
+end

data/bench/codecs/string_hack.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class StringHack
+  class FastIdList
+    def self.from_array(array)
+      new(",#{array.join(",")},")
+    end
+
+    def self.from_string(string)
+      new(string)
+    end
+
+    def initialize(packed)
+      @packed = packed
+    end
+
+    def include?(id)
+      @packed.include?(",#{id},")
+    end
+
+    def to_s
+      @packed
+    end
+  end
+
+  def initialize
+    @oj_options = { mode: :compat }
+  end
+
+  def encode(hash)
+    if user_ids = hash["user_ids"]
+      hash["user_ids"] = FastIdList.from_array(user_ids).to_s
+    end
+
+    ::Oj.dump(hash, @oj_options)
+  end
+
+  def decode(payload)
+    result = ::Oj.load(payload, @oj_options)
+
+    if str = result["user_ids"]
+      result["user_ids"] = FastIdList.from_string(str)
+    end
+
+    result
+  end
+end

data/bench/codecs_large_user_list.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'message_bus', path: '../'
+  gem 'benchmark-ips'
+  gem 'oj'
+end
+
+require 'benchmark/ips'
+require 'message_bus'
+require_relative 'codecs/all_codecs'
+
+bench_decode({
+  "data" => "hello world",
+  "user_ids" => (1..10000).to_a,
+  "group_ids" => nil,
+  "client_ids" => nil
+  }, 5000
+)
+
+# packed_string_4_bytes:   127176.1 i/s
+# packed_string_8_bytes:    94494.6 i/s - 1.35x  (± 0.00) slower
+#          string_hack:    26403.4 i/s - 4.82x  (± 0.00) slower
+#              marshal:     4985.5 i/s - 25.51x  (± 0.00) slower
+#                   oj:     3072.9 i/s - 41.39x  (± 0.00) slower
+#                 json:     2222.7 i/s - 57.22x  (± 0.00) slower

data/bench/codecs_standard_message.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'message_bus', path: '../'
+  gem 'benchmark-ips'
+  gem 'oj'
+end
+
+require 'benchmark/ips'
+require 'message_bus'
+require_relative 'codecs/all_codecs'
+
+bench_decode({
+  "data" => { amazing: "hello world this is an amazing message hello there!!!", another_key: [2, 3, 4] },
+  "user_ids" => [1, 2, 3],
+  "group_ids" => [1],
+  "client_ids" => nil
+  }, 2
+)
+
+#              marshal:   504885.6 i/s
+#                 json:   401050.9 i/s - 1.26x  (± 0.00) slower
+#                   oj:   340847.4 i/s - 1.48x  (± 0.00) slower
+#          string_hack:   296741.6 i/s - 1.70x  (± 0.00) slower
+# packed_string_4_bytes:   207942.6 i/s - 2.43x  (± 0.00) slower
+# packed_string_8_bytes:   206093.0 i/s - 2.45x  (± 0.00) slower

data/lib/message_bus.rb

@@ -2,24 +2,29 @@
 
 require "monitor"
 require "set"
-require "message_bus/version"
-require "message_bus/message"
-require "message_bus/client"
-require "message_bus/connection_manager"
-require "message_bus/diagnostics"
-require "message_bus/rack/middleware"
-require "message_bus/rack/diagnostics"
-require "message_bus/timer_thread"
+
+require_relative "message_bus/version"
+require_relative "message_bus/message"
+require_relative "message_bus/client"
+require_relative "message_bus/connection_manager"
+require_relative "message_bus/diagnostics"
+require_relative "message_bus/rack/middleware"
+require_relative "message_bus/rack/diagnostics"
+require_relative "message_bus/timer_thread"
+require_relative "message_bus/codec/base"
+require_relative "message_bus/backends"
+require_relative "message_bus/backends/base"
 
 # we still need to take care of the logger
-if defined?(::Rails)
-  require 'message_bus/rails/railtie'
+if defined?(::Rails::Engine)
+  require_relative 'message_bus/rails/railtie'
 end
 
 # @see MessageBus::Implementation
 module MessageBus; end
 MessageBus::BACKENDS = {}
 class MessageBus::InvalidMessage < StandardError; end
+class MessageBus::InvalidMessageTarget < MessageBus::InvalidMessage; end
 class MessageBus::BusDestroyed < StandardError; end
 
 # The main server-side interface to a message bus for the purposes of
@@ -277,6 +282,17 @@ module MessageBus::Implementation
       end
   end
 
+  # @param [MessageBus::Codec::Base] codec used to encode and decode Message payloads
+  # @return [void]
+  def transport_codec=(codec)
+    configure(trasport_codec: codec)
+  end
+
+  # @return [MessageBus::Codec::Base] codec used to encode and decode Message payloads
+  def transport_codec
+    @config[:transport_codec] ||= MessageBus::Codec::Json.new
+  end
+
   # @param [MessageBus::Backend::Base] pub_sub a configured backend
   # @return [void]
   def reliable_pub_sub=(pub_sub)
@@ -329,6 +345,7 @@ module MessageBus::Implementation
   #
   # @raise [MessageBus::BusDestroyed] if the bus is destroyed
   # @raise [MessageBus::InvalidMessage] if attempting to put permission restrictions on a globally-published message
+  # @raise [MessageBus::InvalidMessageTarget] if attempting to publish to a empty group of users
   def publish(channel, data, opts = nil)
     return if @off
 
@@ -348,22 +365,32 @@ module MessageBus::Implementation
       site_id = opts[:site_id]
     end
 
-    raise ::MessageBus::InvalidMessage if (user_ids || group_ids) && global?(channel)
+    if (user_ids || group_ids) && global?(channel)
+      raise ::MessageBus::InvalidMessage
+    end
 
-    encoded_data = JSON.dump(
-      data: data,
-      user_ids: user_ids,
-      group_ids: group_ids,
-      client_ids: client_ids
-    )
+    if (user_ids == []) || (group_ids == []) || (client_ids == [])
+      raise ::MessageBus::InvalidMessageTarget
+    end
+
+    encoded_data = transport_codec.encode({
+      "data" => data,
+      "user_ids" => user_ids,
+      "group_ids" => group_ids,
+      "client_ids" => client_ids
+    })
+
+    channel_opts = {}
 
-    channel_opts = nil
+    if opts
+      if ((age = opts[:max_backlog_age]) || (size = opts[:max_backlog_size]))
+        channel_opts[:max_backlog_size] = size
+        channel_opts[:max_backlog_age] = age
+      end
 
-    if opts && ((age = opts[:max_backlog_age]) || (size = opts[:max_backlog_size]))
-      channel_opts = {
-        max_backlog_size: size,
-        max_backlog_age: age
-      }
+      if opts.has_key?(:queue_in_memory)
+        channel_opts[:queue_in_memory] = opts[:queue_in_memory]
+      end
     end
 
     encoded_channel_name = encode_channel_name(channel, site_id)
@@ -614,7 +641,7 @@ module MessageBus::Implementation
     channel, site_id = decode_channel_name(msg.channel)
     msg.channel = channel
     msg.site_id = site_id
-    parsed = JSON.parse(msg.data)
+    parsed = transport_codec.decode(msg.data)
     msg.data = parsed["data"]
     msg.user_ids = parsed["user_ids"]
     msg.group_ids = parsed["group_ids"]

data/lib/message_bus/backends/base.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "message_bus/backends"
-
 module MessageBus
   module Backends
     # Backends provide a consistent API over a variety of options for persisting

data/lib/message_bus/backends/memory.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "message_bus/backends/base"
-
 module MessageBus
   module Backends
     # The memory backend stores published messages in a simple array per

data/lib/message_bus/backends/postgres.rb

@@ -2,8 +2,6 @@
 
 require 'pg'
 
-require "message_bus/backends/base"
-
 module MessageBus
   module Backends
     # The Postgres backend stores published messages in a single Postgres table

data/lib/message_bus/backends/redis.rb

@@ -3,8 +3,6 @@
 require 'redis'
 require 'digest'
 
-require "message_bus/backends/base"
-
 module MessageBus
   module Backends
     # The Redis backend stores published messages in Redis sorted sets (using
@@ -104,8 +102,8 @@ module MessageBus
 
       local global_id = redis.call("INCR", global_id_key)
       local backlog_id = redis.call("INCR", backlog_id_key)
-      local payload = string.format("%i|%i|%s", global_id, backlog_id, start_payload)
-      local global_backlog_message = string.format("%i|%s", backlog_id, channel)
+      local payload = table.concat({ global_id, backlog_id, start_payload }, "|")
+      local global_backlog_message = table.concat({ backlog_id, channel }, "|")
 
       redis.call("ZADD", backlog_key, backlog_id, payload)
       redis.call("EXPIRE", backlog_key, max_backlog_age)
@@ -318,7 +316,7 @@ LUA
             end
           end
         rescue => error
-          @logger.warn "#{error} subscribe failed, reconnecting in 1 second. Call stack #{error.backtrace}"
+          @logger.warn "#{error} subscribe failed, reconnecting in 1 second. Call stack #{error.backtrace.join("\n")}"
           sleep 1
           global_redis&.disconnect!
           retry

data/lib/message_bus/client.rb

@@ -133,7 +133,6 @@ class MessageBus::Client
     user_allowed = false
     group_allowed = false
 
-    # this is an inconsistency we should fix anyway, publishing `user_ids: nil` should work same as groups
     has_users = msg.user_ids && msg.user_ids.length > 0
     has_groups = msg.group_ids && msg.group_ids.length > 0
 

data/lib/message_bus/codec/base.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module MessageBus
+  module Codec
+    class Base
+      def encode(hash)
+        raise ConcreteClassMustImplementError
+      end
+
+      def decode(payload)
+        raise ConcreteClassMustImplementError
+      end
+    end
+
+    autoload :Json, File.expand_path("json", __dir__)
+    autoload :Oj, File.expand_path("oj", __dir__)
+  end
+end

data/lib/message_bus/codec/json.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module MessageBus
+  module Codec
+    class Json < Base
+      def encode(hash)
+        JSON.dump(hash)
+      end
+
+      def decode(payload)
+        JSON.parse(payload)
+      end
+    end
+  end
+end