message_bus 2.0.6 → 2.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e3233d29ea9404805798a4b1a4caddf2a50477c
-  data.tar.gz: f77eed645f31f2182dc28d94fdf80dc8746ffffd
+  metadata.gz: 6261c99f5e5950c4d602faeb54aa133b747075e1
+  data.tar.gz: e6318de70431d0ca0ed84efb14c54379bcccf527
 SHA512:
-  metadata.gz: 4a1b8e23c41ea624f21012e8c9f83be06217d1cfb103dd5c13417aff52bf49fab6dde813185f5186b8bdd226267bd5ac2aec859cdd49c35323e35f8c51a8af57
-  data.tar.gz: b1285bcba99c5263eba003781a8423245d198edb88bf2c1265d895a771d6b75bc1da89e119cb973d08a58fa4c98726d9bb75ed425c8b28f089c5fdbed70db8f1
+  metadata.gz: 3f6a1cd06d53694cf1ccdc7455f1d3adbdee5c2f09ce9b38cd9cec5a26128b1d979b35dea9f3add98143554620171118508d560c057e3380a5c9880a9225348e
+  data.tar.gz: bf4373a4595e43ba6adb59cd3e8046c190a039a055af657fa05fa2f224914a1461c1f0dec30e1ce708fe8c9d518180563270b0fbf9455e5039ffda4379b14b9a

data/.travis.yml

@@ -1,7 +1,6 @@
 before_install: gem install bundler
 language: ruby
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1.8
   - 2.2.4

data/CHANGELOG

@@ -1,3 +1,11 @@
+02-10-2017
+
+- Version 2.0.7
+
+- Fix/Feature: use LUA script for publishing to bus, this eliminates a race condition
+   and ensures that we are never stuck in a multi transaction by mistake
+
+
 29-09-2017
 
 - Version 2.0.6

data/lib/message_bus/backends/redis.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
+#
 require 'redis'
+require 'digest'
+
 # the heart of the message bus, it acts as 2 things
 #
 # 1. A channel multiplexer
@@ -81,46 +84,70 @@ class MessageBus::Redis::ReliablePubSub
     end
   end
 
-  def publish(channel, data, queue_in_memory = true)
-    redis = pub_redis
-    backlog_id_key = backlog_id_key(channel)
-    backlog_key = backlog_key(channel)
+  LUA_PUBLISH = <<LUA
 
-    global_id = nil
-    backlog_id = nil
+  local start_payload = ARGV[1]
+  local max_backlog_age = ARGV[2]
+  local max_backlog_size = tonumber(ARGV[3])
+  local max_global_backlog_size = tonumber(ARGV[4])
+  local channel = ARGV[5]
 
-    redis.multi do |m|
-      global_id = m.incr(global_id_key)
-      backlog_id = m.incr(backlog_id_key)
-    end
+  local global_id_key = KEYS[1]
+  local backlog_id_key = KEYS[2]
+  local backlog_key = KEYS[3]
+  local global_backlog_key = KEYS[4]
+  local redis_channel_name = KEYS[5]
 
-    global_id = global_id.value
-    backlog_id = backlog_id.value
+  local global_id = redis.call("INCR", global_id_key)
+  local backlog_id = redis.call("INCR", backlog_id_key)
+  local payload = string.format("%i|%i|%s", global_id, backlog_id, start_payload)
+  local global_backlog_message = string.format("%i|%s", backlog_id, channel)
 
-    msg = MessageBus::Message.new global_id, backlog_id, channel, data
-    payload = msg.encode
+  redis.call("ZADD", backlog_key, backlog_id, payload)
+  redis.call("EXPIRE", backlog_key, max_backlog_age)
 
-    redis.multi do |m|
+  redis.call("ZADD", global_backlog_key, global_id, global_backlog_message)
+  redis.call("EXPIRE", global_backlog_key, max_backlog_age)
 
-      redis.zadd backlog_key, backlog_id, payload
-      redis.expire backlog_key, @max_backlog_age
+  redis.call("PUBLISH", redis_channel_name, payload)
 
-      redis.zadd global_backlog_key, global_id, backlog_id.to_s << "|" << channel
-      redis.expire global_backlog_key, @max_backlog_age
+  if backlog_id > max_backlog_size then
+    redis.call("ZREMRANGEBYSCORE", backlog_key, 1, backlog_id - max_backlog_size)
+  end
 
-      redis.publish redis_channel_name, payload
+  if global_id > max_global_backlog_size then
+    redis.call("ZREMRANGEBYSCORE", global_backlog_key, 1, global_id - max_global_backlog_size)
+  end
 
-      if backlog_id > @max_backlog_size
-        redis.zremrangebyscore backlog_key, 1, backlog_id - @max_backlog_size
-      end
+  return backlog_id
 
-      if global_id > @max_global_backlog_size
-        redis.zremrangebyscore global_backlog_key, 1, global_id - @max_global_backlog_size
-      end
+LUA
 
-    end
+  LUA_PUBLISH_SHA1 = Digest::SHA1.hexdigest(LUA_PUBLISH)
 
-    backlog_id
+  def publish(channel, data, queue_in_memory = true)
+    redis = pub_redis
+    backlog_id_key = backlog_id_key(channel)
+    backlog_key = backlog_key(channel)
+
+    msg = MessageBus::Message.new nil, nil, channel, data
+
+    cached_eval(redis, LUA_PUBLISH, LUA_PUBLISH_SHA1,
+               argv: [
+                 msg.encode_without_ids,
+                 max_backlog_age,
+                 max_backlog_size,
+                 max_global_backlog_size,
+                 channel
+               ],
+               keys: [
+                 global_id_key,
+                 backlog_id_key,
+                 backlog_key,
+                 global_backlog_key,
+                 redis_channel_name
+               ]
+    )
 
   rescue Redis::CommandError => e
     if queue_in_memory && e.message =~ /^READONLY/
@@ -342,6 +369,18 @@ class MessageBus::Redis::ReliablePubSub
 
   private
 
+  def cached_eval(redis, script, script_sha1, params)
+    begin
+      redis.evalsha script_sha1, params
+    rescue Redis::CommandError => e
+      if e.to_s =~ /^NOSCRIPT/
+        redis.eval script, params
+      else
+        raise
+      end
+    end
+  end
+
   def is_readonly?
     key = "__mb_is_readonly"
 

data/lib/message_bus/message.rb

@@ -17,4 +17,8 @@ class MessageBus::Message < Struct.new(:global_id, :message_id, :channel , :data
   def encode
     global_id.to_s << "|" << message_id.to_s << "|" << channel.gsub("|", "$$123$$") << "|" << data
   end
+
+  def encode_without_ids
+    channel.gsub("|", "$$123$$") << "|" << data
+  end
 end

data/lib/message_bus/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module MessageBus
-  VERSION = "2.0.6"
+  VERSION = "2.0.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: message_bus
 version: !ruby/object:Gem::Version
-  version: 2.0.6
+  version: 2.0.7
 platform: ruby
 authors:
 - Sam Saffron
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-29 00:00:00.000000000 Z
+date: 2017-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack