message_bus-http_client 1.0.0.pre1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: da6aba476a01a5481d92971aab35612b332e3cb2187d5f5b633b7466202f219b
+  data.tar.gz: 5e8fd7f77341e8e156697a193816dd39fee11cd81e650a65c7bff071a911fb88
+SHA512:
+  metadata.gz: e988e4d40e3cb571c7b3aa7c803465317f913d25bf8b06c30c76b6befde831a7bd06e9b4e8141d19f61a4a154a538b475a82ce3e550df90cca6464a5b3a69cc2
+  data.tar.gz: ba35c7f2e3faf4df33d190d59182bd9c673ba50304b9a8d49a5bb8085c40bb023dc94888da5466f90b2363ee4a6335ff889521d425ff3c904dcbc764bb442a05

data/lib/message_bus/http_client.rb

@@ -0,0 +1,294 @@
+require 'securerandom'
+require 'net/http'
+require 'json'
+require 'uri'
+require 'message_bus/http_client/configuration'
+require 'message_bus/http_client/channel'
+
+module MessageBus
+  # MessageBus client that enables subscription via long polling with support
+  # for chunked encoding. Falls back to normal polling if long polling is not
+  # available.
+  #
+  # @!attribute [r] channels
+  #   @return [Hash] a map of the channels that the client is subscribed to
+  # @!attribute [r] stats
+  #   @return [Stats] a Struct containing the statistics of failed and successful
+  #     polling requests
+  class HTTPClient
+    include Configuration
+
+    class InvalidChannel < StandardError; end
+
+    attr_reader :channels, :stats
+
+    CHUNK_SEPARATOR = "\r\n|\r\n".freeze
+    private_constant :CHUNK_SEPARATOR
+    STATUS_CHANNEL = "/__status".freeze
+    private_constant :STATUS_CHANNEL
+
+    STOPPED = 0
+    STARTED = 1
+
+    Stats = Struct.new(:failed, :success)
+    private_constant :Stats
+
+    # @param base_url [String] Base URL of the message_bus server to connect to
+    #
+    # @return [Object] Instance of MessageBus::HTTPClient
+    def initialize(base_url, *args)
+      super
+      @uri = URI(base_url)
+      @client_id = SecureRandom.hex
+      @channels = {}
+      @status = STOPPED
+      @mutex = Mutex.new
+      @stats = Stats.new(0, 0)
+    end
+
+    # Starts a background thread that polls the message bus endpoint
+    # for the given base_url.
+    #
+    # Intervals for long polling can be configured via min_poll_interval and
+    # max_poll_interval.
+    #
+    # Intervals for polling can be configured via background_callback_interval.
+    #
+    # @return [Object] Instance of MessageBus::HTTPClient
+    def start
+      @mutex.synchronize do
+        return if started?
+
+        @status = STARTED
+
+        thread = Thread.new do
+          begin
+            while started?
+              poll unless @channels.empty?
+              @stats.success += 1
+              sleep interval
+            end
+          rescue StandardError => e
+            @stats.failed += 1
+            warn("#{e.class} #{e.message}: #{e.backtrace.join("\n")}")
+            retry
+          ensure
+            stop
+          end
+        end
+
+        thread.abort_on_exception = true
+      end
+
+      self
+    end
+
+    # Stops the client from polling the message bus endpoint.
+    #
+    # @return [Integer] the current status of the client
+    def stop
+      @status = STOPPED
+    end
+
+    # Subscribes to a channel which executes the given callback when a message
+    # is published to the channel
+    #
+    # @example Subscribing to a channel for message
+    #   client = MessageBus::HTTPClient.new('http://some.test.com')
+    #
+    #   client.subscribe("/test") do |payload, _message_id, _global_id|
+    #     puts payload
+    #   end
+    #
+    # A last_message_id may be provided.
+    #  * -1 will subscribe to all new messages
+    #  * -2 will recieve last message + all new messages
+    #  * -3 will recieve last 2 message + all new messages
+    #
+    # @example Subscribing to a channel with `last_message_id`
+    #   client.subscribe("/test", last_message_id: -2) do |payload|
+    #     puts payload
+    #   end
+    #
+    # @param channel [String] channel to listen for messages on
+    # @param last_message_id [Integer] last message id to start polling on.
+    #
+    # @yield [data, message_id, global_id]
+    #  callback to be executed whenever a message is received
+    #
+    # @yieldparam data [Hash] data payload of the message received on the channel
+    # @yieldparam message_id [Integer] id of the message in the channel
+    # @yieldparam global_id [Integer] id of the message in the global backlog
+    # @yieldreturn [void]
+    #
+    # @return [Integer] the current status of the client
+    def subscribe(channel, last_message_id: nil, &callback)
+      raise InvalidChannel unless channel.to_s.start_with?("/")
+
+      last_message_id = -1 if last_message_id && !last_message_id.is_a?(Integer)
+
+      @channels[channel] ||= Channel.new
+      channel = @channels[channel]
+      channel.last_message_id = last_message_id if last_message_id
+      channel.callbacks.push(callback)
+      start if stopped?
+    end
+
+    # unsubscribes from a channel
+    #
+    # @example Unsubscribing from a channel
+    #   client = MessageBus::HTTPClient.new('http://some.test.com')
+    #   callback = -> { |payload| puts payload }
+    #   client.subscribe("/test", &callback)
+    #   client.unsubscribe("/test")
+    #
+    # If a callback is given, only the specific callback will be unsubscribed.
+    #
+    # @example Unsubscribing a callback from a channel
+    #   client.unsubscribe("/test", &callback)
+    #
+    # When the client does not have any channels left, it will stop polling and
+    # waits until a new subscription is started.
+    #
+    # @param channel [String] channel to unsubscribe
+    # @yield [data, global_id, message_id] specific callback to unsubscribe
+    #
+    # @return [Integer] the current status of the client
+    def unsubscribe(channel, &callback)
+      if callback
+        @channels[channel].callbacks.delete(callback)
+        remove_channel(channel) if @channels[channel].callbacks.empty?
+      else
+        remove_channel(channel)
+      end
+
+      stop if @channels.empty?
+      @status
+    end
+
+    private
+
+    def stopped?
+      @status == STOPPED
+    end
+
+    def started?
+      @status == STARTED
+    end
+
+    def remove_channel(channel)
+      @channels.delete(channel)
+    end
+
+    def interval
+      if @enable_long_polling
+        if (failed_count = @stats.failed) > 2
+          (@min_poll_interval * failed_count).clamp(
+            @min_poll_interval, @max_poll_interval
+          )
+        else
+          @min_poll_interval
+        end
+      else
+        @background_callback_interval
+      end
+    end
+
+    def poll
+      http = Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = true if @uri.scheme == 'https'
+      request = Net::HTTP::Post.new(request_path, headers)
+      request.body = poll_payload
+
+      if @enable_long_polling
+        buffer = ''
+
+        http.request(request) do |response|
+          response.read_body do |chunk|
+            unless chunk.empty?
+              buffer << chunk
+              process_buffer(buffer)
+            end
+          end
+        end
+      else
+        response = http.request(request)
+        notify_channels(JSON.parse(response.body))
+      end
+    end
+
+    def is_chunked?
+      !headers["Dont-Chunk"]
+    end
+
+    def process_buffer(buffer)
+      index = buffer.index(CHUNK_SEPARATOR)
+
+      if is_chunked?
+        return unless index
+
+        messages = buffer[0..(index - 1)]
+        buffer.slice!("#{messages}#{CHUNK_SEPARATOR}")
+      else
+        messages = buffer[0..-1]
+        buffer.slice!(messages)
+      end
+
+      notify_channels(JSON.parse(messages))
+    end
+
+    def notify_channels(messages)
+      messages.each do |message|
+        current_channel = message['channel']
+
+        if current_channel == STATUS_CHANNEL
+          message["data"].each do |channel_name, last_message_id|
+            if (channel = @channels[channel_name])
+              channel.last_message_id = last_message_id
+            end
+          end
+        else
+          @channels.each do |channel_name, channel|
+            next unless channel_name == current_channel
+
+            channel.last_message_id = message['message_id']
+
+            channel.callbacks.each do |callback|
+              callback.call(
+                message['data'],
+                channel.last_message_id,
+                message['global_id']
+              )
+            end
+          end
+        end
+      end
+    end
+
+    def poll_payload
+      payload = {}
+
+      @channels.each do |channel_name, channel|
+        payload[channel_name] = channel.last_message_id
+      end
+
+      payload.to_json
+    end
+
+    def request_path
+      "/message-bus/#{@client_id}/poll"
+    end
+
+    def headers
+      headers = {}
+      headers['Content-Type'] = 'application/json'
+      headers['X-Silence-logger'] = 'true'
+
+      if !@enable_long_polling || !@enable_chunked_encoding
+        headers['Dont-Chunk'] = 'true'
+      end
+
+      headers.merge!(@headers)
+    end
+  end
+end

data/lib/message_bus/http_client/channel.rb

@@ -0,0 +1,13 @@
+module MessageBus
+  class HTTPClient
+    # @private
+    class Channel
+      attr_accessor :last_message_id, :callbacks
+
+      def initialize(last_message_id: -1, callbacks: [])
+        @last_message_id = last_message_id
+        @callbacks = callbacks
+      end
+    end
+  end
+end

data/lib/message_bus/http_client/configuration.rb

@@ -0,0 +1,40 @@
+module MessageBus
+  class HTTPClient
+    module Configuration
+      def self.included(base)
+        base.send(:attr_accessor,
+          :enable_long_polling,
+          :status,
+          :enable_chunked_encoding,
+          :min_poll_interval,
+          :max_poll_interval,
+          :background_callback_interval
+        )
+      end
+
+      # @param enable_long_polling [Boolean] Enable long polling
+      # @param enable_chunked_encoding [Boolean] Enable chunk encoding
+      # @param min_poll_interval [Float, Integer] Min poll interval when long polling
+      # @param max_poll_interval [Float, Integer] Max poll interval when long polling.
+      #   When requests fail, the client will backoff and this is the upper limit.
+      # @param background_callback_interval [Float, Integer] Interval to poll when
+      #   when polling.
+      # @param headers [Hash] extra HTTP headers to be set on the polling requests.
+      def initialize(_base_url,
+                     enable_long_polling: true,
+                     enable_chunked_encoding: true,
+                     min_poll_interval: 0.1,
+                     max_poll_interval: 180,
+                     background_callback_interval: 60,
+                     headers: {})
+
+        @enable_long_polling = enable_long_polling
+        @enable_chunked_encoding = enable_chunked_encoding
+        @min_poll_interval = min_poll_interval
+        @max_poll_interval = max_poll_interval
+        @background_callback_interval = background_callback_interval
+        @headers = headers
+      end
+    end
+  end
+end

data/lib/message_bus/http_client/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module MessageBus
+  class HTTPClient
+    VERSION = '1.0.0.pre1'
+  end
+end

data/spec/integration/http_client_spec.rb

@@ -0,0 +1,183 @@
+require_relative '../spec_helper'
+require 'message_bus/http_client'
+
+describe MessageBus::HTTPClient do
+  let(:base_url) { "http://0.0.0.0:9292" }
+  let(:client) { MessageBus::HTTPClient.new(base_url) }
+  let(:headers) { client.send(:headers) }
+  let(:channel) { "/test" }
+  let(:channel2) { "/test2" }
+
+  def publish_message
+    response = Net::HTTP.get_response(URI("#{base_url}/publish"))
+    assert_equal("200", response.code)
+  end
+
+  before do
+    @threads = Thread.list
+  end
+
+  after do
+    new_threads = Thread.list - @threads
+    client.stop
+    assert(new_threads.size <= 1)
+    new_threads.each(&:join)
+  end
+
+  describe '#start and #stop' do
+    it 'should be able to start and stop polling correctly' do
+      threads = Thread.list
+
+      assert_equal(MessageBus::HTTPClient::STOPPED, client.status)
+
+      client.start
+      new_threads = Thread.list - threads
+
+      assert_equal(1, new_threads.size)
+      assert_equal(MessageBus::HTTPClient::STARTED, client.status)
+
+      client.start
+
+      assert_equal(new_threads, Thread.list - threads)
+    end
+  end
+
+  describe '#subscribe' do
+    it 'should be able to subscribe to channels for messages' do
+      called = 0
+      called2 = 0
+
+      client.subscribe(channel, last_message_id: -1) do |data|
+        called += 1
+        assert_equal("world", data["hello"])
+      end
+
+      client.subscribe(channel2) do |data|
+        called2 += 1
+        assert_equal("world", data["hello"])
+      end
+
+      while called < 2 && called2 < 2
+        publish_message
+        sleep 0.05
+      end
+    end
+
+    describe 'supports including extra headers' do
+      let(:client) do
+        MessageBus::HTTPClient.new(base_url, headers: {
+          'Dont-Chunk' => "true"
+        })
+      end
+
+      it 'should include the header in the request' do
+        called = 0
+
+        client.subscribe(channel) do |data|
+          called += 1
+          assert_equal("world", data["hello"])
+        end
+
+        while called < 2
+          publish_message
+          sleep 0.05
+        end
+      end
+    end
+
+    describe 'when chunked encoding is disabled' do
+      let(:client) do
+        MessageBus::HTTPClient.new(base_url, enable_chunked_encoding: false)
+      end
+
+      it 'should still be able to subscribe to channels for messages' do
+        called = 0
+
+        client.subscribe(channel) do |data|
+          called += 1
+          assert_equal("world", data["hello"])
+        end
+
+        while called < 2
+          publish_message
+          sleep 0.05
+        end
+      end
+    end
+
+    describe 'when enable_long_polling is disabled' do
+      let(:client) do
+        MessageBus::HTTPClient.new(base_url,
+                                   enable_long_polling: false,
+                                   background_callback_interval: 0.01)
+      end
+
+      it 'should still be able to subscribe to channels for messages' do
+        called = 0
+
+        client.subscribe(channel) do |data|
+          called += 1
+          assert_equal("world", data["hello"])
+        end
+
+        while called < 2
+          publish_message
+          sleep 0.05
+        end
+      end
+    end
+
+    describe 'when channel name is invalid' do
+      it 'should raise the right error' do
+        ["test", 1, :test].each do |invalid_channel|
+          assert_raises MessageBus::HTTPClient::InvalidChannel do
+            client.subscribe(invalid_channel)
+          end
+        end
+      end
+    end
+
+    describe 'with last_message_id' do
+      describe 'when invalid' do
+        it 'should subscribe from the latest message' do
+          client.subscribe(channel, last_message_id: 'haha')
+          assert_equal(-1, client.channels[channel].last_message_id)
+        end
+      end
+
+      describe 'when valid' do
+        it 'should subscribe from the right message' do
+          client.subscribe(channel, last_message_id: -2)
+          assert_equal(-2, client.channels[channel].last_message_id)
+        end
+      end
+    end
+  end
+
+  describe '#unsubscribe' do
+    it 'should be able to unsubscribe a channel' do
+      client.subscribe(channel) { raise "Not called" }
+      assert(client.channels[channel])
+
+      client.unsubscribe(channel)
+      assert_nil(client.channels[channel])
+    end
+
+    describe 'with callback' do
+      it 'should be able to unsubscribe a callback for a particular channel' do
+        callback = -> { raise "Not called" }
+        callback2 = -> { raise "Not called2" }
+
+        client.subscribe(channel, &callback)
+        client.subscribe(channel, &callback2)
+        assert_equal([callback, callback2], client.channels[channel].callbacks)
+
+        client.unsubscribe(channel, &callback)
+        assert_equal([callback2], client.channels[channel].callbacks)
+
+        client.unsubscribe(channel, &callback2)
+        assert_nil(client.channels[channel])
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: message_bus-http_client
+version: !ruby/object:Gem::Version
+  version: 1.0.0.pre1
+platform: ruby
+authors:
+- Tan Guo Xiang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-12-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+description: A Ruby HTTP client for message bus
+email:
+- tgx+github@discourse.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/message_bus/http_client.rb
+- lib/message_bus/http_client/channel.rb
+- lib/message_bus/http_client/configuration.rb
+- lib/message_bus/http_client/version.rb
+- spec/integration/http_client_spec.rb
+homepage: https://github.com/SamSaffron/message_bus
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: ''
+test_files:
+- spec/integration/http_client_spec.rb