message_bus 3.3.4 → 3.3.8

data/assets/message-bus.js

@@ -1,9 +1,9 @@
-/*jshint bitwise: false*/
+/*global define, jQuery*/
 
 (function (root, factory) {
-  if (typeof define === 'function' && define.amd) {
+  if (typeof define === "function" && define.amd) {
     // AMD. Register as an anonymous module.
-    define([], function (b) {
+    define([], function () {
       // Also create a global in case some scripts
       // that are loaded still are looking for
       // a global even when an AMD loader is in use.
@@ -13,12 +13,12 @@
     // Browser globals
     root.MessageBus = factory();
   }
-}(typeof self !== 'undefined' ? self : this, function () {
+})(typeof self !== "undefined" ? self : this, function () {
   "use strict";
 
   // http://stackoverflow.com/questions/105034/how-to-create-a-guid-uuid-in-javascript
-  var uniqueId = function() {
-    return "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, function(c) {
+  var uniqueId = function () {
+    return "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, function (c) {
       var r = (Math.random() * 16) | 0;
       var v = c === "x" ? r : (r & 0x3) | 0x8;
       return v.toString(16);
@@ -31,8 +31,6 @@
   var started = false;
   var clientId = uniqueId();
   var callbacks = [];
-  var queue = [];
-  var interval = null;
   var failCount = 0;
   var baseUrl = "/";
   var paused = false;
@@ -44,7 +42,7 @@
   var totalAjaxCalls = 0;
   var lastAjax;
 
-  var isHidden = (function() {
+  var isHidden = (function () {
     var prefixes = ["", "webkit", "ms", "moz"];
     var hiddenProperty;
     for (var i = 0; i < prefixes.length; i++) {
@@ -55,7 +53,7 @@
       }
     }
 
-    return function() {
+    return function () {
       if (hiddenProperty !== undefined) {
         return document[hiddenProperty];
       } else {
@@ -64,7 +62,7 @@
     };
   })();
 
-  var hasLocalStorage = (function() {
+  var hasLocalStorage = (function () {
     try {
       localStorage.setItem("mbTestLocalStorage", Date.now());
       localStorage.removeItem("mbTestLocalStorage");
@@ -74,13 +72,13 @@
     }
   })();
 
-  var updateLastAjax = function() {
+  var updateLastAjax = function () {
     if (hasLocalStorage) {
       localStorage.setItem("__mbLastAjax", Date.now());
     }
   };
 
-  var hiddenTabShouldWait = function() {
+  var hiddenTabShouldWait = function () {
     if (hasLocalStorage && isHidden()) {
       var lastAjaxCall = parseInt(localStorage.getItem("__mbLastAjax"), 10);
       var deltaAjax = Date.now() - lastAjaxCall;
@@ -91,20 +89,21 @@
   };
 
   var hasonprogress = new XMLHttpRequest().onprogress === null;
-  var allowChunked = function() {
+  var allowChunked = function () {
     return me.enableChunkedEncoding && hasonprogress;
   };
 
-  var shouldLongPoll = function() {
+  var shouldLongPoll = function () {
     return (
       me.alwaysLongPoll ||
       (me.shouldLongPollCallback ? me.shouldLongPollCallback() : !isHidden())
     );
   };
 
-  var processMessages = function(messages) {
-    var gotData = false;
-    if ((!messages) || (messages.length === 0)) { return false; }
+  var processMessages = function (messages) {
+    if (!messages || messages.length === 0) {
+      return false;
+    }
 
     for (var i = 0; i < messages.length; i++) {
       var message = messages[i];
@@ -136,7 +135,7 @@
     return true;
   };
 
-  var reqSuccess = function(messages) {
+  var reqSuccess = function (messages) {
     failCount = 0;
     if (paused) {
       if (messages) {
@@ -150,7 +149,7 @@
     return false;
   };
 
-  var longPoller = function(poll, data) {
+  var longPoller = function (poll, data) {
     if (ajaxInProgress) {
       // never allow concurrent ajax reqs
       return;
@@ -183,7 +182,7 @@
 
     var dataType = chunked ? "text" : "json";
 
-    var handle_progress = function(payload, position) {
+    var handle_progress = function (payload, position) {
       var separator = "\r\n|\r\n";
       var endChunk = payload.indexOf(separator, position);
 
@@ -206,31 +205,13 @@
       return handle_progress(payload, endChunk + separator.length);
     };
 
-    var disableChunked = function() {
+    var disableChunked = function () {
       if (me.longPoll) {
         me.longPoll.abort();
         chunkedBackoff = 30;
       }
     };
 
-    var setOnProgressListener = function(xhr) {
-      var position = 0;
-      // if it takes longer than 3000 ms to get first chunk, we have some proxy
-      // this is messing with us, so just backoff from using chunked for now
-      var chunkedTimeout = setTimeout(disableChunked, 3000);
-      xhr.onprogress = function() {
-        clearTimeout(chunkedTimeout);
-        if (
-          xhr.getResponseHeader("Content-Type") ===
-          "application/json; charset=utf-8"
-        ) {
-          // not chunked we are sending json back
-          chunked = false;
-          return;
-        }
-        position = handle_progress(xhr.responseText, position);
-      };
-    };
     if (!me.ajax) {
       throw new Error("Either jQuery or the ajax adapter must be loaded");
     }
@@ -246,19 +227,18 @@
         "/poll" +
         (!longPoll ? "?dlp=t" : ""),
       data: data,
-      cache: false,
       async: true,
       dataType: dataType,
       type: "POST",
       headers: headers,
       messageBus: {
         chunked: chunked,
-        onProgressListener: function(xhr) {
+        onProgressListener: function (xhr) {
           var position = 0;
           // if it takes longer than 3000 ms to get first chunk, we have some proxy
           // this is messing with us, so just backoff from using chunked for now
           var chunkedTimeout = setTimeout(disableChunked, 3000);
-          return (xhr.onprogress = function() {
+          return (xhr.onprogress = function () {
             clearTimeout(chunkedTimeout);
             if (
               xhr.getResponseHeader("Content-Type") ===
@@ -269,9 +249,9 @@
               position = handle_progress(xhr.responseText, position);
             }
           });
-        }
+        },
       },
-      xhr: function() {
+      xhr: function () {
         var xhr = jQuery.ajaxSettings.xhr();
         if (!chunked) {
           return xhr;
@@ -279,7 +259,7 @@
         this.messageBus.onProgressListener(xhr);
         return xhr;
       },
-      success: function(messages) {
+      success: function (messages) {
         if (!chunked) {
           // we may have requested text so jQuery will not parse
           if (typeof messages === "string") {
@@ -288,7 +268,7 @@
           gotData = reqSuccess(messages);
         }
       },
-      error: function(xhr, textStatus, err) {
+      error: function (xhr, textStatus) {
         if (xhr.status === 429) {
           var tryAfter =
             parseInt(
@@ -307,7 +287,7 @@
           totalAjaxFailures += 1;
         }
       },
-      complete: function() {
+      complete: function () {
         ajaxInProgress = false;
 
         var interval;
@@ -345,14 +325,14 @@
         }
 
         if (started) {
-          pollTimeout = setTimeout(function() {
+          pollTimeout = setTimeout(function () {
             pollTimeout = null;
             poll();
           }, interval);
         }
 
         me.longPoll = null;
-      }
+      },
     });
 
     return req;
@@ -374,7 +354,7 @@
     baseUrl: baseUrl,
     headers: {},
     ajax: typeof jQuery !== "undefined" && jQuery.ajax,
-    diagnostics: function() {
+    diagnostics: function () {
       console.log("Stopped: " + stopped + " Started: " + started);
       console.log("Current callbacks");
       console.log(callbacks);
@@ -391,42 +371,50 @@
       );
     },
 
-    pause: function() {
+    pause: function () {
       paused = true;
     },
 
-    resume: function() {
+    resume: function () {
       paused = false;
       processMessages(later);
       later = [];
     },
 
-    stop: function() {
+    stop: function () {
       stopped = true;
       started = false;
       if (delayPollTimeout) {
         clearTimeout(delayPollTimeout);
         delayPollTimeout = null;
       }
+      if (pollTimeout) {
+        clearTimeout(pollTimeout);
+        pollTimeout = null;
+      }
       if (me.longPoll) {
         me.longPoll.abort();
       }
+      if (me.onVisibilityChange) {
+        document.removeEventListener("visibilitychange", me.onVisibilityChange);
+        me.onVisibilityChange = null;
+      }
     },
 
     // Start polling
-    start: function() {
+    start: function () {
       if (started) return;
       started = true;
       stopped = false;
 
-      var poll = function() {
+      var poll = function () {
         if (stopped) {
           return;
         }
 
         if (callbacks.length === 0 || hiddenTabShouldWait()) {
           if (!delayPollTimeout) {
-            delayPollTimeout = setTimeout(function() {
+            delayPollTimeout = setTimeout(function () {
               delayPollTimeout = null;
               poll();
             }, parseInt(500 + Math.random() * 500));
@@ -448,25 +436,28 @@
 
       // monitor visibility, issue a new long poll when the page shows
       if (document.addEventListener && "hidden" in document) {
-        me.visibilityEvent = document.addEventListener(
-          "visibilitychange",
-          function() {
-            if (!document.hidden && !me.longPoll && pollTimeout) {
-              clearTimeout(pollTimeout);
-              clearTimeout(delayPollTimeout);
-
-              delayPollTimeout = null;
-              pollTimeout = null;
-              poll();
-            }
+        me.onVisibilityChange = function () {
+          if (
+            !document.hidden &&
+            !me.longPoll &&
+            (pollTimeout || delayPollTimeout)
+          ) {
+            clearTimeout(pollTimeout);
+            clearTimeout(delayPollTimeout);
+
+            delayPollTimeout = null;
+            pollTimeout = null;
+            poll();
           }
-        );
+        };
+
+        document.addEventListener("visibilitychange", me.onVisibilityChange);
       }
 
       poll();
     },
 
-    status: function() {
+    status: function () {
       if (paused) {
         return "paused";
       } else if (started) {
@@ -485,7 +476,7 @@
     // -2 will recieve last message + all new messages
     // -3 will recieve last 2 messages + all new messages
     // if undefined will default to -1
-    subscribe: function(channel, func, lastId) {
+    subscribe: function (channel, func, lastId) {
       if (!started && !stopped) {
         me.start();
       }
@@ -493,7 +484,9 @@
       if (lastId === null || typeof lastId === "undefined") {
         lastId = -1;
       } else if (typeof lastId !== "number") {
-        throw "lastId has type " + typeof lastId + " but a number was expected.";
+        throw (
+          "lastId has type " + typeof lastId + " but a number was expected."
+        );
       }
 
       if (typeof channel !== "string") {
@@ -503,7 +496,7 @@
       callbacks.push({
         channel: channel,
         func: func,
-        last_id: lastId
+        last_id: lastId,
       });
       if (me.longPoll) {
         me.longPoll.abort();
@@ -513,7 +506,7 @@
     },
 
     // Unsubscribe from a channel
-    unsubscribe: function(channel, func) {
+    unsubscribe: function (channel, func) {
       // TODO allow for globbing in the middle of a channel name
       // like /something/*/something
       // at the moment we only support globbing /something/*
@@ -550,7 +543,7 @@
       }
 
       return removed;
-    }
+    },
   };
   return me;
-}));
+});

data/bench/codecs/all_codecs.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative './packed_string'
+require_relative './string_hack'
+require_relative './marshal'
+
+def all_codecs
+  {
+    json: MessageBus::Codec::Json.new,
+    oj: MessageBus::Codec::Oj.new,
+    marshal: MarshalCodec.new,
+    packed_string_4_bytes: PackedString.new("V"),
+    packed_string_8_bytes: PackedString.new("Q"),
+    string_hack: StringHack.new
+  }
+end
+
+def bench_decode(hash, user_needle)
+  encoded_data = all_codecs.map do |name, codec|
+    [
+      name, codec, codec.encode(hash.dup)
+    ]
+  end
+
+  Benchmark.ips do |x|
+
+    encoded_data.each do |name, codec, encoded|
+      x.report(name) do |n|
+        while n > 0
+          decoded = codec.decode(encoded)
+          decoded["user_ids"].include?(user_needle)
+          n -= 1
+        end
+      end
+    end
+
+    x.compare!
+  end
+end

data/bench/codecs/marshal.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class MarshalCodec
+  def encode(hash)
+    ::Marshal.dump(hash)
+  end
+
+  def decode(payload)
+    ::Marshal.load(payload) # rubocop:disable Security/MarshalLoad
+  end
+end

data/bench/codecs/packed_string.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class PackedString
+  class FastIdList
+    def self.from_array(array, pack_with)
+      new(array.sort.pack("#{pack_with}*"), pack_with)
+    end
+
+    def self.from_string(string, pack_with)
+      new(string, pack_with)
+    end
+
+    def initialize(packed, pack_with)
+      raise "unknown pack format, expecting Q or V" if pack_with != "V" && pack_with != "Q"
+      @packed = packed
+      @pack_with = pack_with
+      @slot_size = pack_with == "V" ? 4 : 8
+    end
+
+    def include?(id)
+      found = (0...length).bsearch do |index|
+        @packed.byteslice(index * @slot_size, @slot_size).unpack1(@pack_with) >= id
+      end
+
+      found && @packed.byteslice(found * @slot_size, @slot_size).unpack1(@pack_with) == id
+    end
+
+    def length
+      @length ||= @packed.bytesize / @slot_size
+    end
+
+    def to_a
+      @packed.unpack("#{@pack_with}*")
+    end
+
+    def to_s
+      @packed
+    end
+  end
+
+  def initialize(pack_with = "V")
+    @pack_with = pack_with
+    @oj_options = { mode: :compat }
+  end
+
+  def encode(hash)
+
+    if user_ids = hash["user_ids"]
+      hash["user_ids"] = FastIdList.from_array(hash["user_ids"], @pack_with).to_s
+    end
+
+    hash["data"] = ::Oj.dump(hash["data"], @oj_options)
+
+    Marshal.dump(hash)
+  end
+
+  def decode(payload)
+    result = Marshal.load(payload) # rubocop:disable Security/MarshalLoad
+    result["data"] = ::Oj.load(result["data"], @oj_options)
+
+    if str = result["user_ids"]
+      result["user_ids"] = FastIdList.from_string(str, @pack_with)
+    end
+
+    result
+  end
+end

data/bench/codecs/string_hack.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class StringHack
+  class FastIdList
+    def self.from_array(array)
+      new(",#{array.join(",")},")
+    end
+
+    def self.from_string(string)
+      new(string)
+    end
+
+    def initialize(packed)
+      @packed = packed
+    end
+
+    def include?(id)
+      @packed.include?(",#{id},")
+    end
+
+    def to_s
+      @packed
+    end
+  end
+
+  def initialize
+    @oj_options = { mode: :compat }
+  end
+
+  def encode(hash)
+    if user_ids = hash["user_ids"]
+      hash["user_ids"] = FastIdList.from_array(user_ids).to_s
+    end
+
+    ::Oj.dump(hash, @oj_options)
+  end
+
+  def decode(payload)
+    result = ::Oj.load(payload, @oj_options)
+
+    if str = result["user_ids"]
+      result["user_ids"] = FastIdList.from_string(str)
+    end
+
+    result
+  end
+end

data/bench/codecs_large_user_list.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'message_bus', path: '../'
+  gem 'benchmark-ips'
+  gem 'oj'
+end
+
+require 'benchmark/ips'
+require 'message_bus'
+require_relative 'codecs/all_codecs'
+
+bench_decode({
+  "data" => "hello world",
+  "user_ids" => (1..10000).to_a,
+  "group_ids" => nil,
+  "client_ids" => nil
+  }, 5000
+)
+
+# packed_string_4_bytes:   127176.1 i/s
+# packed_string_8_bytes:    94494.6 i/s - 1.35x  (± 0.00) slower
+#          string_hack:    26403.4 i/s - 4.82x  (± 0.00) slower
+#              marshal:     4985.5 i/s - 25.51x  (± 0.00) slower
+#                   oj:     3072.9 i/s - 41.39x  (± 0.00) slower
+#                 json:     2222.7 i/s - 57.22x  (± 0.00) slower

data/bench/codecs_standard_message.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'message_bus', path: '../'
+  gem 'benchmark-ips'
+  gem 'oj'
+end
+
+require 'benchmark/ips'
+require 'message_bus'
+require_relative 'codecs/all_codecs'
+
+bench_decode({
+  "data" => { amazing: "hello world this is an amazing message hello there!!!", another_key: [2, 3, 4] },
+  "user_ids" => [1, 2, 3],
+  "group_ids" => [1],
+  "client_ids" => nil
+  }, 2
+)
+
+#              marshal:   504885.6 i/s
+#                 json:   401050.9 i/s - 1.26x  (± 0.00) slower
+#                   oj:   340847.4 i/s - 1.48x  (± 0.00) slower
+#          string_hack:   296741.6 i/s - 1.70x  (± 0.00) slower
+# packed_string_4_bytes:   207942.6 i/s - 2.43x  (± 0.00) slower
+# packed_string_8_bytes:   206093.0 i/s - 2.45x  (± 0.00) slower

data/examples/bench/bench.lua

@@ -1,9 +1,9 @@
 -- wrk returns lots of read errors, this is unavoidable cause
 --
--- 1. There is no internal implmentation of chunked encoding in wrk (which would be ideal)
+-- 1. There is no internal implementation of chunked encoding in wrk (which would be ideal)
 --
 -- 2. MessageBus gem does not provide http keepalive (by design), and can not provide content length
---   if MessageBus provided keepalive it would have to be able to redispatch the reqs to rack, something
+--   if MessageBus provided keepalive it would have to be able to re-dispatch requests to rack, something
 --   that is not supported by the underlying rack hijack protocol, once a req is hijacked it can not be
 --   returned
 --

data/lib/message_bus/backends/base.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "message_bus/backends"
-
 module MessageBus
   module Backends
     # Backends provide a consistent API over a variety of options for persisting
@@ -64,7 +62,7 @@ module MessageBus
       attr_accessor :max_backlog_size
       # @return [Integer] the largest permitted size (number of messages) for the global backlog; beyond this capacity, old messages will be dropped.
       attr_accessor :max_global_backlog_size
-      # @return [Integer] the longest amount of time a message may live in a backlog before beging removed, in seconds.
+      # @return [Integer] the longest amount of time a message may live in a backlog before being removed, in seconds.
       attr_accessor :max_backlog_age
       # Typically, backlogs are trimmed whenever we publish to them. This setting allows some tolerance in order to improve performance.
       # @return [Integer] the interval of publications between which the backlog will not be cleared.
@@ -76,7 +74,7 @@ module MessageBus
       # @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
       def initialize(config = {}, max_backlog_size = 1000); end
 
-      # Performs routines specific to the backend that are necessary after a process fork, typically triggerd by a forking webserver. Typically this re-opens sockets to the backend.
+      # Performs routines specific to the backend that are necessary after a process fork, typically triggered by a forking webserver. Typically this re-opens sockets to the backend.
       def after_fork
         raise ConcreteClassMustImplementError
       end
@@ -86,6 +84,11 @@ module MessageBus
         raise ConcreteClassMustImplementError
       end
 
+      # Closes all open connections to the storage.
+      def destroy
+        raise ConcreteClassMustImplementError
+      end
+
       # Deletes all backlogs and their data. Does not delete non-backlog data that message_bus may persist, depending on the concrete backend implementation. Use with extreme caution.
       # @abstract
       def expire_all_backlogs!
@@ -98,7 +101,7 @@ module MessageBus
       # @param [JSON] data some data to publish to the channel. Must be an object that can be encoded as JSON
       # @param [Hash] opts
       # @option opts [Boolean] :queue_in_memory (true) whether or not to hold the message in an in-memory buffer if publication fails, to be re-tried later
-      # @option opts [Integer] :max_backlog_age (`self.max_backlog_age`) the longest amount of time a message may live in a backlog before beging removed, in seconds
+      # @option opts [Integer] :max_backlog_age (`self.max_backlog_age`) the longest amount of time a message may live in a backlog before being removed, in seconds
       # @option opts [Integer] :max_backlog_size (`self.max_backlog_size`) the largest permitted size (number of messages) for the channel backlog; beyond this capacity, old messages will be dropped
       #
       # @return [Integer] the channel-specific ID the message was given