redisse 0.4.0

data/example/spec/app_spec.rb

@@ -0,0 +1,39 @@
+require 'rack/test'
+
+$app, _opts = Rack::Builder.parse_file __dir__ + '/../config.ru'
+
+describe "Example App" do
+  include Rack::Test::Methods
+
+  def app
+    $app
+  end
+
+  describe "/publish" do
+    context "basic" do
+      before do
+        Redisse.test_mode!
+      end
+
+      it "publishes the message to the channel" do
+        post "/publish", channel: 'global', message: 'Hello'
+        expect(Redisse.published.size).to be == 1
+        event = Redisse.published.first
+        expect(event.channel).to be == 'global'
+        expect(event.type).to be == :message
+        expect(event.data).to be == 'Hello'
+      end
+    end
+
+    context "filtered" do
+      before do
+        Redisse.test_filter = :unused_type
+      end
+
+      it "publishes the message with the 'message' type" do
+        post "/publish", channel: 'global', message: 'Hello'
+        expect(Redisse.published.size).to be == 0
+      end
+    end
+  end
+end

data/lib/redisse.rb

@@ -0,0 +1,181 @@
+require 'redisse/version'
+require 'redisse/publisher'
+require 'redis'
+
+# Public: A HTTP API to serve Server-Sent Events via a Redis backend.
+module Redisse
+  # Public: Gets/Sets the String URL of the Redis server to connect to.
+  #
+  # Note that while the Redis pubsub mechanism works outside of the Redis key
+  # namespace and ignores the database (the path part of the URL), the
+  # database will still be used to store an history of the events sent to
+  # support Last-Event-Id.
+  #
+  # Defaults to the REDISSE_REDIS environment variable and if it is not set, to
+  # redis://localhost:6379/.
+  attr_accessor :redis_server
+
+  # Public: The port on which the server listens.
+  #
+  # Defaults to the REDISSE_PORT environment variable and if it is not set, to
+  # 8080.
+  attr_accessor :default_port
+
+  # Public: The internal URL hierarchy to redirect to with X-Accel-Redirect.
+  #
+  # When this property is set, Redisse will work totally differently. Your Ruby
+  # code will not be loaded by the events server itself, but only by the
+  # {#redirect_endpoint} Rack app that you will have to route to in your Rack
+  # app (e.g. using +map+ in +config.ru+) and this endpoint will redirect to
+  # this internal URL hierarchy.
+  #
+  # Defaults to /redisse.
+  attr_accessor :nginx_internal_url
+
+  # Public: Send an event to subscribers, of the given type.
+  #
+  # All browsers subscribing to the events server will receive a Server-Sent
+  # Event of the chosen type.
+  #
+  # channel      - The channel to publish the message to.
+  # type_message - The type of the event and the content of the message, as a
+  #                Hash of form { type => message } or simply the message as
+  #                a String, for the default event type :message.
+  #
+  # Examples
+  #
+  #   Redisse.publish(:global, notice: 'This is a server-sent event.')
+  #   Redisse.publish(:global, 'Hello, World!')
+  #
+  #   # on the browser side:
+  #   var source = new EventSource(eventsURL);
+  #   source.addEventListener('notice', function(e) {
+  #     console.log(e.data) // logs 'This is a server-sent event.'
+  #   }, false)
+  #   source.addEventListener('message', function(e) {
+  #     console.log(e.data) // logs 'Hello, World!'
+  #   }, false)
+  def publish(channel, message)
+    type, message = Hash(message).first if message.respond_to?(:to_h)
+    type ||= :message
+    publisher.publish(channel, message, type)
+  end
+
+  # Public: The list of channels to subscribe to.
+  #
+  # Once {Redisse.channels} has been called, the given block is this method.
+  # The block must satisfy this interface:
+  #
+  # env - The Rack environment for this request.
+  #
+  # Returns an Array of String naming the channels to subscribe to.
+  #
+  # Raises NotImplementedError unless {Redisse.channels} has been called.
+  def channels(env)
+    raise NotImplementedError, "you must call Redisse.channels first"
+  end
+
+  # Public: Use test mode.
+  #
+  # Instead of actually publishing to Redis, events will be stored in
+  # {#published} to use for tests.
+  #
+  # Must be called before each test in order for published events to be
+  # emptied.
+  #
+  # See also {#test_filter=}.
+  #
+  # Examples
+  #
+  #   # RSpec
+  #   before { Redisse.test_mode! }
+  def test_mode!
+    @publisher = TestPublisher.new
+  end
+
+  # Public: Filter events stored in test mode.
+  #
+  # If set, only events whose type match with the filter are stored in
+  # {#published}. A filter matches by using case equality, which allows using
+  # a simple Symbol or a Proc for more advanced filters:
+  #
+  # Automatically sets {#test_mode!}, so it also clears the previous events.
+  #
+  # Examples
+  #
+  #   Redisse.test_filter = -> type { %i(foo baz).include? type }
+  #   Redisse.publish :global, foo: 'stored'
+  #   Redisse.publish :global, bar: 'skipped'
+  #   Redisse.publish :global, baz: 'stored'
+  #   Redisse.published.size # => 2
+  def test_filter=(filter)
+    test_mode!
+    publisher.filter = filter
+  end
+
+  # Public: Returns the published events.
+  #
+  # Fails unless {#test_mode!} is set.
+  def published
+    fail "Call #{self}.test_mode! first" unless publisher.respond_to?(:published)
+    publisher.published
+  end
+
+  # Internal: List of middlewares defined with {#use}.
+  #
+  # Used by Goliath to build the server.
+  def middlewares
+    @middlewares ||= []
+  end
+
+  # Public: Define a middleware for the server.
+  #
+  # See {https://github.com/postrank-labs/goliath/wiki/Middleware Goliath middlewares}.
+  #
+  # Examples
+  #
+  #    Redisse.use MyMiddleware, foo: true
+  def use(middleware, *args, &block)
+    middlewares << [middleware, args, block]
+  end
+
+  # Public: Define a Goliath plugin to run with the server.
+  #
+  # See {https://github.com/postrank-labs/goliath/wiki/Plugins Goliath plugins}.
+  def plugin(name, *args)
+    plugins << [name, args]
+  end
+
+  # Public: The Rack application that redirects to {#nginx_internal_url}.
+  #
+  # If you set {#nginx_internal_url}, you need to call this Rack application
+  # to redirect to the Redisse server.
+  #
+  # Also note that when using the redirect endpoint, two channel names are
+  # reserved, and cannot be used: +polling+ and +lastEventId+.
+  #
+  # Examples
+  #
+  #    map "/events" { run Redisse.redirect_endpoint }
+  def redirect_endpoint
+    @redirect_endpoint ||= RedirectEndpoint.new self
+  end
+
+  autoload :RedirectEndpoint, __dir__ + '/redisse/redirect_endpoint'
+
+private
+
+  def plugins
+    @plugins ||= []
+  end
+
+  def publisher
+    @publisher ||= RedisPublisher.new(redis)
+  end
+
+  def redis
+    @redis ||= Redis.new(url: redis_server)
+  end
+end
+
+require 'redisse/configuration'

data/lib/redisse/configuration.rb

@@ -0,0 +1,47 @@
+module Redisse
+  extend self
+
+  # Public: Define the list of channels to subscribe to.
+  #
+  # Calls the given block with a Rack environment, the block is expected to
+  # return a list of channels the current user has access to. The list is then
+  # coerced using +Kernel#Array+.
+  #
+  # Once the block is defined, other calls will be handled by the block
+  # directly, as if the method had been redefined directly. It simply gives a
+  # nicer API:
+  #
+  #   Redisse.channels do |env|
+  #   end
+  #
+  # vs
+  #
+  #   def Redisse.channels(env)
+  #   end
+  #
+  # block - The block that lists the channels for the given Rack environment.
+  #
+  # Examples
+  #
+  #   Redisse.channels do |env|
+  #     %w( comment post )
+  #   end
+  #   # will result in subscriptions to 'comment' and 'post' channels.
+  #
+  #   Redisse.channels({})
+  #   # => ["comment", "post"]
+  def self.channels(*, &block)
+    if block
+      # overwrite method with block
+      define_singleton_method :channels, &block
+    else
+      super
+    end
+  end
+
+  self.redis_server = ENV['REDISSE_REDIS'] ||
+    'redis://localhost:6379/'
+  self.default_port = ENV['REDISSE_PORT'] ||
+    8080
+  self.nginx_internal_url = '/redisse'
+end

data/lib/redisse/publisher.rb

@@ -0,0 +1,70 @@
+require 'redisse/server_sent_events'
+require 'json'
+
+module Redisse
+  # Internal: Publisher that pushes to Redis with history.
+  class RedisPublisher
+    include ServerSentEvents
+
+    REDISSE_LAST_EVENT_ID = 'redisse:lastEventId'.freeze
+    HISTORY_SIZE = 100
+
+    def initialize(redis)
+      @redis = redis or raise 'RedisPublisher needs a Redis client'
+    end
+
+    def publish(channel, data, type)
+      event_id = @redis.incr(REDISSE_LAST_EVENT_ID)
+      event = server_sent_event(data, type: type, id: event_id)
+      @redis.publish(channel, event)
+      @redis.zadd(channel, event_id, event)
+      @redis.zremrangebyrank(channel, 0, -1-HISTORY_SIZE)
+      event_id
+    end
+  end
+
+  # Internal: Publisher that stores events in memory for easy testing.
+  #
+  # See {Redisse#test_mode! Redisse#test_mode!}.
+  class TestPublisher
+    def initialize
+      @published = []
+    end
+
+    attr_reader :published
+
+    attr_accessor :filter
+
+    def publish(channel, data, type)
+      return if filter && !(filter === type)
+      @published << TestEvent.new(channel, data, type)
+    end
+  end
+
+  # Define then reopen instead of using the block of Struct.new for YARD.
+  TestEvent = Struct.new :channel, :data, :type
+
+  # Public: An event in test mode.
+  #
+  # You can re-open or add modules to this class if you want to add behavior
+  # to events found in {Redisse#published Redisse#published} for easier
+  # testing.
+  #
+  # Examples
+  #
+  #   class Redisse::TestEvent
+  #     def yml
+  #       YAML.load data
+  #     end
+  #
+  #     def private?
+  #       channel.start_with? 'private'
+  #     end
+  #   end
+  class TestEvent
+    # Public: Helper method to parse the Event data as JSON.
+    def json
+      JSON.parse(data)
+    end
+  end
+end

data/lib/redisse/redirect_endpoint.rb

@@ -0,0 +1,45 @@
+require 'uri'
+
+module Redisse
+
+  # Public: Rack app that redirects to the Redisse server via X-Accel-Redirect.
+  class RedirectEndpoint
+
+    def initialize(redisse)
+      @redisse = redisse
+      self.base_url = redisse.nginx_internal_url
+    end
+
+    def call(env)
+      response = Rack::Response.new
+      response['X-Accel-Redirect'] = redirect_url(env)
+      response
+    end
+
+  private
+
+    def redirect_url(env)
+      channels = @redisse.channels(env)
+      fail 'Wrong channel "polling"' if channels.include? 'polling'
+      fail 'Reserved channel "lastEventId"' if channels.include? 'lastEventId'
+      @base_url + '?' + URI.encode_www_form(redirect_options(env) + channels)
+    end
+
+    def redirect_options(env)
+      params = URI.decode_www_form(env['QUERY_STRING'])
+      [].tap do |options|
+        options << 'polling'.freeze if params.assoc('polling'.freeze)
+        last_event_id_param = params.assoc('lastEventId')
+        options << last_event_id_param if last_event_id_param
+      end
+    end
+
+    def base_url=(url)
+      url = String(url)
+      url += "/" unless url.end_with? '/'
+      @base_url = url
+    end
+
+  end
+
+end

data/lib/redisse/server.rb

@@ -0,0 +1,205 @@
+require 'redisse'
+require 'goliath/api'
+require 'rack/accept_media_types'
+require 'goliath/runner'
+require 'em-hiredis'
+
+module Redisse
+
+  # Public: Run the server.
+  #
+  # If you use the provided binary you don't need to call this method.
+  #
+  # By default, the {#channels} method is called directly.
+  #
+  # If {#nginx_internal_url} is set, the channels will actually come from the
+  # internal redirect URL generated in the Rack app by {#redirect_endpoint}.
+  def run
+    run_as_standalone if nginx_internal_url
+    server = Server.new(self)
+    runner = Goliath::Runner.new(ARGV, server)
+    runner.app = Goliath::Rack::Builder.build(self, server)
+    runner.load_plugins([Server::Stats] + plugins)
+    runner.run
+  end
+
+private
+
+  # Internal: Redefine {#channels} to find channels in the redirect URL.
+  def run_as_standalone
+    channels do |env|
+      query_string = env['QUERY_STRING'] || ''
+      channels = query_string.split('&').map { |channel|
+        URI.decode_www_form_component(channel)
+      }
+      channels.delete('polling')
+      channels.delete_if {|channel| channel.start_with?('lastEventId=') }
+    end
+  end
+
+  # Internal: Goliath::API class that defines the server.
+  #
+  # See {Redisse#run}.
+  class Server < Goliath::API
+    require 'redisse/server/stats'
+    require 'redisse/server/responses'
+    include Responses
+    require 'redisse/server/redis'
+    include Redis
+
+    # Public: Delay between receiving a message and closing the connection.
+    #
+    # Closing the connection is necessary when using long polling, because the
+    # client is not able to read the data before the connection is closed. But
+    # instead of closing immediately, we delay a bit closing the connection to
+    # give a chance for several messages to be sent in a row.
+    LONG_POLLING_DELAY = 1
+
+    # Public: The period between heartbeats in seconds.
+    HEARTBEAT_PERIOD = 15
+
+    def initialize(redisse)
+      @redisse = redisse
+      super()
+    end
+
+    def response(env)
+      return not_acceptable unless acceptable?(env)
+      channels = Array(redisse.channels(env))
+      return not_found if channels.empty?
+      subscribe(env, channels) or return service_unavailable
+      send_history_events(env, channels)
+      heartbeat(env)
+      streaming_response(200, {
+        'Content-Type' => 'text/event-stream',
+        'Cache-Control' => 'no-cache',
+        'X-Accel-Buffering' => 'no',
+      })
+    end
+
+    def on_close(env)
+      env.status[:stats][:connected] -= 1
+      env.status[:stats][:served]    += 1
+      unsubscribe(env)
+      stop_heartbeat(env)
+    end
+
+  private
+
+    attr_reader :redisse
+
+    def subscribe(env, channels)
+      return unless pubsub { env.stream_close }
+      env.status[:stats][:connected] += 1
+      env.logger.debug { "Subscribing to #{channels}" }
+      env_sender = -> event { send_event(env, event) }
+      pubsub_subcribe(channels, env_sender)
+      env['redisse.unsubscribe'.freeze] = -> do
+        pubsub_unsubscribe_proc(channels, env_sender)
+      end
+      true
+    end
+
+    def heartbeat(env)
+      env['redisse.heartbeat_timer'.freeze] = EM.add_periodic_timer(HEARTBEAT_PERIOD) do
+        env.logger.debug "Sending heartbeat".freeze
+        env.stream_send(": hb\n".freeze)
+      end
+    end
+
+    def stop_heartbeat(env)
+      return unless timer = env['redisse.heartbeat_timer'.freeze]
+      env.logger.debug "Stopping heartbeat".freeze
+      timer.cancel
+    end
+
+    def unsubscribe(env)
+      return unless unsubscribe = env['redisse.unsubscribe'.freeze]
+      env['redisse.unsubscribe'.freeze] = nil
+      env.logger.debug "Unsubscribing".freeze
+      unsubscribe.call
+    end
+
+    def send_event(env, event)
+      env.status[:stats][:events] += 1
+      env.logger.debug { "Sending:\n#{event.chomp.chomp}" }
+      env.stream_send(event)
+      return unless long_polling?(env)
+      env["redisse.long_polling_timer".freeze] ||= EM.add_timer(LONG_POLLING_DELAY) do
+        env.stream_close
+      end
+    end
+
+    def long_polling?(env)
+      key = "redisse.long_polling".freeze
+      env.fetch(key) do
+        env[key] = Rack::Request.new(env).GET.keys.include?('polling')
+      end
+    end
+
+    def send_history_events(env, channels)
+      last_event_id = last_event_id(env)
+      return unless last_event_id
+      EM::Synchrony.next_tick do
+        events = events_for_channels(channels, last_event_id)
+        env.logger.debug { "Sending #{events.size} history events" }
+        if (first = events.first) && first.start_with?('type: missedevents')
+          env.status[:stats][:missing] += 1
+        end
+        events.each { |event| send_event(env, event) }
+      end
+    end
+
+    def last_event_id(env)
+      last_event_id = env['HTTP_LAST_EVENT_ID'] ||
+        Rack::Request.new(env).GET['lastEventId']
+      last_event_id = last_event_id.to_i
+      last_event_id.nonzero? && last_event_id
+    end
+
+    def events_for_channels(channels, last_event_id)
+      events_with_ids = channels.each_with_object([]) { |channel, events|
+        channel_events = events_for_channel(channel, last_event_id)
+        events.concat(channel_events)
+      }.sort_by!(&:last)
+      handle_missing_events(events_with_ids, last_event_id)
+      events_with_ids.map(&:first)
+    end
+
+    def handle_missing_events(events_with_ids, last_event_id)
+      first_event, first_event_id = events_with_ids.first
+      return unless first_event
+      if first_event_id == last_event_id
+        events_with_ids.shift
+      else
+        event = ServerSentEvents.server_sent_event(nil, type: :missedevents)
+        events_with_ids.unshift([event])
+      end
+    end
+
+    def events_for_channel(channel, last_event_id)
+      df = redis.zrangebyscore(channel, last_event_id, '+inf', 'withscores')
+      events_scores = EM::Synchrony.sync(df)
+      events_scores.each_slice(2).map do |event, score|
+        [event, score.to_i]
+      end
+    end
+
+    def acceptable?(env)
+      accept_media_types = Rack::AcceptMediaTypes.new(env['HTTP_ACCEPT'])
+      accept_media_types.include?('text/event-stream')
+    end
+
+  public
+
+    def options_parser(opts, options)
+      opts.on '--redis REDIS_URL', 'URL of the Redis connection' do |url|
+        redisse.redis_server = url
+      end
+      default_port = redisse.default_port
+      return unless default_port
+      options[:port] = default_port
+    end
+
+  end
+end