pubsubstub 0.0.15 → 0.1.0.beta1

data/lib/pubsubstub/subscriber.rb

@@ -0,0 +1,88 @@
+module Pubsubstub
+  class Subscriber
+    include Logging
+    include Mutex_m
+
+    def initialize
+      super
+      @subscribed = false
+      @listeners = {}
+    end
+
+    def subscribed?
+      @subscribed
+    end
+
+    def add_event_listener(channel_key, callback)
+      synchronize do
+        @listeners[channel_key] ||= Set.new
+        !!@listeners[channel_key].add?(callback)
+      end
+    end
+
+    def remove_event_listener(channel_key, callback)
+      synchronize do
+        return unless @listeners[channel_key]
+        !!@listeners[channel_key].delete?(callback)
+      end
+    end
+
+    def stop
+      # redis.client.call allow to bypass the client mutex
+      # Since we now that the only other possible caller is blocking on reading the socket this is safe
+      synchronize do
+        redis.client.call(['punsubscribe', pubsub_pattern])
+      end
+    end
+
+    def start
+      redis.psubscribe(pubsub_pattern) do |on|
+        on.psubscribe do
+          info { "Subscribed to #{pubsub_pattern}" }
+          @subscribed = true
+        end
+
+        on.punsubscribe do
+          info { "Unsubscribed from #{pubsub_pattern}" }
+          @subscribed = false
+        end
+
+        on.pmessage do |pattern, pubsub_key, message|
+          process_message(pubsub_key, message)
+        end
+      end
+    ensure
+      info { "Terminated" }
+    end
+
+    private
+
+    def pubsub_pattern
+      '*.pubsub'
+    end
+
+    def process_message(pubsub_key, message)
+      channel_name = Channel.name_from_pubsub_key(pubsub_key)
+      event = Event.from_json(message)
+      dispatch_event(channel_name, event)
+    end
+
+    def dispatch_event(channel_name, event)
+      listeners = listeners_for(channel_name)
+      info { "Dispatching event ##{event.id} from #{channel_name} to #{listeners.size} listeners" }
+      listeners.each do |listener|
+        listener.call(event)
+      end
+    end
+
+    def listeners_for(channel_name)
+      @listeners.fetch(channel_name) { [] }
+    end
+
+    # The Redis client suround all calls with a mutex.
+    # As such it is crucial to use a dedicated Redis client when blocking on a `subscribe` call.
+    def redis
+      @redis ||= Pubsubstub.new_redis
+    end
+  end
+end

data/lib/pubsubstub/subscription.rb

@@ -0,0 +1,62 @@
+module Pubsubstub
+  class Subscription
+    include Logging
+
+    attr_reader :channels, :connection, :queue, :id
+
+    def initialize(channels, connection)
+      @id = Random.rand(2 ** 64)
+      @connection = connection
+      @channels = channels
+      @queue = Queue.new
+    end
+
+    def stream(last_event_id)
+      info { "Connecting client ##{id} (#{channels.map(&:name).join(', ')})" }
+      subscribe
+      fetch_scrollback(last_event_id)
+      while event = queue.pop
+        debug { "Sending event ##{event.id} to client ##{id}"}
+        connection << event.to_message
+      end
+    ensure
+      info { "Disconnecting client ##{id}" }
+      unsubscribe
+    end
+
+    def push(event)
+      queue.push(event)
+    end
+
+    private
+
+    def subscribe
+      channels.each { |c| Pubsubstub.subscriber.add_event_listener(c.name, callback) }
+    end
+
+    def unsubscribe
+      channels.each { |c| Pubsubstub.subscriber.remove_event_listener(c.name, callback) }
+    end
+
+    # This method is not ideal as it doesn't guarantee order in case of multi-channel subscription
+    def fetch_scrollback(last_event_id)
+      event_sent = false
+      if last_event_id
+        channels.each do |channel|
+          channel.scrollback(since: last_event_id).each do |event|
+            event_sent = true
+            queue.push(event)
+          end
+        end
+      end
+
+      queue.push(Pubsubstub.heartbeat_event) unless event_sent
+    end
+
+    # We use store the callback so that the object_id stays the same.
+    # Otherwise we wouldn't be able to unsubscribe
+    def callback
+      @callback ||= method(:push)
+    end
+  end
+end

data/lib/pubsubstub/version.rb

@@ -1,3 +1,3 @@
 module Pubsubstub
-  VERSION = "0.0.15"
+  VERSION = "0.1.0.beta1"
 end

data/pubsubstub.gemspec

@@ -19,16 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'sinatra', "~> 1.4"
-  spec.add_dependency 'em-hiredis', "~> 0.2"
   spec.add_dependency 'redis', "~> 3.0"
-  spec.add_dependency 'eventmachine', "~> 1.0"
 
   spec.add_development_dependency "bundler", "~> 1.5"
-  spec.add_development_dependency "rake", "~> 10.2"
-  spec.add_development_dependency "rspec", "3.1.0"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "thin", "~> 1.6"
-  spec.add_development_dependency "rack-test"
-  spec.add_development_dependency "timecop"
-  spec.add_development_dependency "em-spec"
 end

data/spec/channel_spec.rb

@@ -1,93 +1,39 @@
 require 'spec_helper'
 
 describe Pubsubstub::Channel do
-  let(:pubsub) { double(Pubsubstub::RedisPubSub, publish: true, subscribe: true, unsubscribe: true) }
-  subject { Pubsubstub::Channel.new("test") }
-  before { allow(subject).to receive(:pubsub) { pubsub } }
+  subject { described_class.new('foobar') }
 
-  context "#initialize" do
-    it "does not subscribe immediately" do
-      expect(pubsub).not_to receive(:subscribe)
-      Pubsubstub::Channel.new('test')
-    end
+  it "has a name" do
+    expect(subject.name).to be == 'foobar'
   end
 
-  context "#subscribe" do
-    let(:connection) { double('connection') }
-    it "subscribes the client" do
-      subject.subscribe(connection)
-      expect(subject.subscribed?(connection)).to be true
-    end
-
-    it "starts subscribing to the channel if it's the first client" do
-      expect(pubsub).to receive(:subscribe)
-      subject.subscribe(connection)
-    end
-
-    it "does not starts subscribing to the channel if it's not the first client" do
-      subject.subscribe(connection)
-      expect(pubsub).not_to receive(:subscribe)
-      subject.subscribe(double('connection'))
-    end
-
-    it "sends the scrollback if a last_event_id is passed" do
-      event = Pubsubstub::Event.new("event")
-      expect(pubsub).to receive(:scrollback).with(1234).and_yield(event)
-      expect(connection).to receive(:<<).with(event.to_message)
-      subject.subscribe(connection, last_event_id: 1234)
-    end
+  it "has a scrollback key derived from the name" do
+    expect(subject.scrollback_key).to be == 'foobar.scrollback'
   end
 
-  context "#unsubscribe" do
-    let(:connection) { double('connection') }
-    before { subject.subscribe(connection) }
-
-    it "unsubscribes the client" do
-      expect(subject.subscribed?(connection)).to be true
-      subject.unsubscribe(connection)
-      expect(subject.subscribed?(connection)).to be false
-    end
-
-    it "does not stop listening if it's not the last client" do
-      subject.subscribe(double('connection'))
-      expect(pubsub).not_to receive(:unsubscribe)
-      subject.unsubscribe(connection)
-    end
-
-    it "stops listening if it's the last client" do
-      expect(pubsub).to receive(:unsubscribe)
-      subject.unsubscribe(connection)
-    end
+  it "has a pubsub key derived from the name" do
+    expect(subject.pubsub_key).to be == 'foobar.pubsub'
   end
 
-  context "#publish" do
-    it "forwards to the pubsub" do
-      event = double('event')
-      expect(pubsub).to receive(:publish).with(event)
-      subject.publish(event)
+  describe "#publish" do
+    let(:event) { Pubsubstub::Event.new("refresh #1", id: 1) }
+
+    it "records the event in the scrollback" do
+      expect {
+        subject.publish(event)
+      }.to change { subject.scrollback(since: 0) }.from([]).to([event])
     end
   end
 
-  context "broadcasting events from redis" do
-    let(:event) { Pubsubstub::Event.new("message", name: "toto") }
-    let(:connection) { double('connection') }
-    before {
-      subject.subscribe(connection)
-    }
-
-    it "sends the events to the clients" do
-      expect(connection).to receive(:<<).with(event.to_message)
-      subject.send(:broadcast, event.to_json)
+  describe "#scrollback" do
+    before do
+      10.times do |i|
+        subject.publish(Pubsubstub::Event.new("refresh ##{i}", id: i))
+      end
     end
-  end
 
-  context "#scrollback" do
-    it "sends events to the connection buffer" do
-      event = Pubsubstub::Event.new("message")
-      expect(pubsub).to receive(:scrollback).and_yield(event)
-      connection = ""
-      subject.scrollback(connection, 1)
-      expect(connection).to eq(event.to_message)
+    it "returns the events with an id greater than the `since` parameter" do
+      expect(subject.scrollback(since: 5).map(&:id)).to be == [6, 7, 8, 9]
     end
   end
 end

data/spec/publish_action_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Pubsubstub::StreamAction do
+  let(:app) { Pubsubstub::PublishAction.new }
+  let(:channel) { Pubsubstub::Channel.new('foo') }
+
+  it "adds the event to the scrollback" do
+    expect {
+      post '/?channels[]=foo', {'event' => 'hello', 'data' => 'world!'}
+      expect(last_response.status).to eq 200
+    }.to change { channel.scrollback(since: 0).size }.from(0).to(1)
+  end
+end

data/spec/spec_helper.rb

@@ -1,14 +1,33 @@
+require 'open3'
 require 'rack/test'
 require 'pry'
 require 'pry-byebug'
 require 'timecop'
-require 'em-spec/rspec'
+require 'thread'
+
+require_relative 'support/threading_matchers'
+require_relative 'support/http_helpers'
+
+Thread.abort_on_exception = true # ensure no exception stays hidden in threads
 
 ENV['RACK_ENV'] = 'test'
 require_relative '../lib/pubsubstub'
 
+Pubsubstub.logger = Logger.new(nil)
+Pubsubstub.logger.level = Logger::DEBUG
+
+# Fake EM
+module EventMachine
+  extend self
+  
+  def reactor_running?
+    false
+  end
+end
+
 RSpec.configure do |config|
   config.include Rack::Test::Methods
+  config.include HTTPHelpers
 
   config.run_all_when_everything_filtered = true
   config.filter_run :focus
@@ -16,6 +35,5 @@ RSpec.configure do |config|
 
   config.order = 'random'
 
-  config.before(:each) { Pubsubstub::RedisPubSub.blocking_redis.flushdb }
+  config.before(:each) { Redis.new(url: Pubsubstub.redis_url).flushdb }
 end
-

data/spec/stream_action_spec.rb

@@ -1,119 +1,56 @@
 require 'spec_helper'
 
-describe "Pubsubstub::StreamAction without EventMachine" do
-  before {
-    allow(EventMachine).to receive(:reactor_running?).and_return(false)
-  }
-
+describe Pubsubstub::StreamAction do
   let(:app) { Pubsubstub::StreamAction.new }
-  it "returns a heartbeat if there is no LAST_EVENT_ID" do
-    Timecop.freeze(DateTime.parse("2015-01-01T00:00:00+00:00")) do
-      event = Pubsubstub::Event.new(
-        'ping',
-        name: 'heartbeat',
-        retry_after: Pubsubstub::StreamAction::RECONNECT_TIMEOUT,
-      )
-      get "/"
-      expect(last_response.body).to eq(event.to_message)
-    end
-  end
-
-  it "returns an empty body if a LAST_EVENT_ID is provided and there is no scrollback" do
-    get "/", {}, 'HTTP_LAST_EVENT_ID' => 1
-    expect(last_response.body).to eq('')
-  end
-
-  it "returns the content of the scrollback" do
-    event = Pubsubstub::Event.new("test")
-    Pubsubstub::RedisPubSub.publish(:default, event)
-
-    get "/", {}, 'HTTP_LAST_EVENT_ID' => 1
-    expect(last_response.body).to eq(event.to_message)
-  end
-end
-
-describe "Pubsubstub::StreamAction with EventMachine" do
-  include EventMachine::SpecHelper
 
-  let(:app) {
-    Pubsubstub::StreamAction.new
-  }
-
-  it "returns a heartbeat if there is no LAST_EVENT_ID" do
-    Timecop.freeze(DateTime.parse("2015-01-01T00:00:00+00:00")) do
-      event = Pubsubstub::Event.new(
-        'ping',
-        name: 'heartbeat',
-        retry_after: Pubsubstub::StreamAction::RECONNECT_TIMEOUT,
-      )
-      get "/"
-      expect(last_response.body).to eq(event.to_message)
+  context "with EventMachine" do
+    before do
+      allow(EventMachine).to receive(:reactor_running?).and_return(true)
     end
-  end
-
-  it "returns the content of the scrollback right away" do
-    event = Pubsubstub::Event.new("test")
-    Pubsubstub::RedisPubSub.publish(:default, event)
-    expect_any_instance_of(EventMachine::Hiredis::Client).to receive(:zrangebyscore).and_yield([event.to_json])
 
-    em do
-      env = current_session.send(:env_for, "/", 'HTTP_LAST_EVENT_ID' => 1)
-      request = Rack::Request.new(env)
-      status, headers, body = app.call(request.env)
+    it "immediately returns the scrollback" do
+      Pubsubstub.publish('foo', 'bar', id: 1)
+      Pubsubstub.publish('foo', 'baz', id: 2)
 
-      response = Rack::MockResponse.new(status, headers, body, env["rack.errors"].flush)
-
-      EM.next_tick {
-        body.close
-        response.finish
+      get '/?channels[]=foo', {}, 'HTTP_LAST_EVENT_ID' => 1
+      expect(last_response.body).to eq("id: 2\ndata: baz\n\n")
+    end
 
-        expect(response.body).to eq(event.to_message)
-        EM.stop
-      }
+    it "returns and heartbeat if scrollback is empty" do
+      Timecop.freeze('2015-01-01T00:00:00+00:00') do
+        get '/'
+        message = "id: 1420070400000\nevent: heartbeat\nretry: #{Pubsubstub.reconnect_timeout}\ndata: ping\n\n"
+        expect(last_response.body).to eq(message)
+      end
     end
   end
 
-  it "subscribes the connection to the channel" do
-    em do
-      redis = spy('Redis Pubsub')
-      allow(Pubsubstub::RedisPubSub).to receive(:sub).and_return(redis)
-      expect(redis).to receive(:subscribe).with("default.pubsub", anything)
+  it "immediately send a heartbeat event if there is no scrollback" do
+    with_background_server do
+      expect(9292).to listen.in_under(5)
 
-      env = current_session.send(:env_for, "/", 'HTTP_LAST_EVENT_ID' => 1)
-      request = Rack::Request.new(env)
-      status, headers, body = app.call(request.env)
+      chunks = async_get('http://localhost:9292/?channels[]=foo', 'Last-Event-Id' => '0')
+      expect(chunks.pop).to include("event: heartbeat\n")
 
-      response = Rack::MockResponse.new(status, headers, body, env["rack.errors"].flush)
+      Pubsubstub.publish('foo', 'bar', id: 1)
+      expect(chunks.pop).to include("id: 1\n")
 
-      EM.next_tick {
-        body.close
-        response.finish
-        EM.stop
-      }
+      Pubsubstub.publish('foo', 'baz', id: 2)
+      expect(chunks.pop).to include("id: 2\n")
     end
   end
 
-  it "sends heartbeat events every now and then" do
-    allow(Pubsubstub).to receive(:heartbeat_frequency).and_return(0.001)
-
-    Timecop.freeze do
-      em do
-        env = current_session.send(:env_for, "/", 'HTTP_LAST_EVENT_ID' => 1)
-        request = Rack::Request.new(env)
-        status, headers, body = app.call(request.env)
+  it "sends the scrollback if there is some" do
+    Pubsubstub.publish('foo', 'bar', id: 1)
 
-        response = Rack::MockResponse.new(status, headers, body, env["rack.errors"].flush)
+    with_background_server do
+      expect(9292).to listen.in_under(5)
 
-        event = Pubsubstub::Event.new('ping', name: 'heartbeat', retry_after: Pubsubstub::StreamAction::RECONNECT_TIMEOUT)
+      chunks = async_get('http://localhost:9292/?channels[]=foo', 'Last-Event-Id' => '0')
+      expect(chunks.pop).to include("id: 1\n")
 
-        EM.add_timer(0.002) {
-          body.close
-          response.finish
-
-          expect(response.body).to start_with(event.to_message)
-          EM.stop
-        }
-      end
+      Pubsubstub.publish('foo', 'baz', id: 2)
+      expect(chunks.pop).to include("id: 2\n")
     end
   end
-end
+end