pubsubstub 0.0.15 → 0.1.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e14d40f41e5f06347d7f552009f53d29bb3d4942
-  data.tar.gz: e1fca82791cb2f1769aa8bf3a6076736b70f8e4c
+  metadata.gz: 66e90ee4856de125eb629c5d0b5eac58fde45493
+  data.tar.gz: f3a7ff09c124a70524fc1d8e31615a51ab53c1e2
 SHA512:
-  metadata.gz: dad828aea534513d6acef76af10e057e8ad21f8804c1fccc9102ce1c1db8149dedd5e74f9730d4a4d64a67b1815fdc05b70149e2f9f561b90582cd9ac1c3fc7d
-  data.tar.gz: a9afa981566c4320fd578e99dbb843e537e2e1f10d668ab735929ec96dcc34a9a99eb0bad3c013594a378f544c130cb8ffbfce6700b42270005e5a03218d79e6
+  metadata.gz: 1d2004519f159a56587cef4a57dd88ff7eec7215a27a247a04ca30442821dc82ec0528e28743b198ae233cd04f1eba9c4df98bee4033f6a0c1646a50c565dc6f
+  data.tar.gz: b27b55471cf1ece69e956c6eb2e8606bfe65576bfdaa9236d02007b1ce371e1985d94e7b4de74035b6d1d7d6e69d23c395e052db020bdd6efc529042e1a55a0a

data/.travis.yml

@@ -1,6 +1,8 @@
 language: ruby
 rvm:
   - 2.2.2
-script: rspec spec
 services:
   - redis-server
+
+after_failure:
+  - cat tmp/puma.log

data/Gemfile

@@ -2,3 +2,11 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in pubsubstub.gemspec
 gemspec
+
+gem "rspec", "3.1.0"
+gem "pry-byebug"
+gem "puma", "~> 3.4"
+gem "thin", "~> 1.6"
+gem "rack-test"
+gem "timecop"
+gem "rake", "~> 10.0"

data/README.md

@@ -56,12 +56,18 @@ run Pubsubstub::Application
 ### Sending an event
 
 ```ruby
-payload = user.to_json
-event = Pubsubstub::Event.new(payload, name: "user.update")
-Pubsubstub::RedisPubSub.publish("user.#{user_id}", event)
+Pubsubstub.publish("user.#{user.id}", user.to_json, name: "user.update")
 ```
 
-To start the application, run `bundle exec thin start --timeout 0 --max-conns 1024`
+To start the application, run `bundle exec puma config.ru`
+
+### HTTP Server
+
+It is heavilly recommended to deploy Pubsubstub with `puma >= 3.4.0`. As of April 2016, it is the only ruby server that properly handle persistent connections.
+
+Other servers like thin will either require you to set a connection timeout otherwise you won't be able to shutdown the server properly.
+
+See the [/example/puma_config.rb](example puma config) for how to configure puma properly.
 
 ## Contributing
 

data/Rakefile

@@ -1 +1,6 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'console' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("pubsubstub", "console")

data/bin/server

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec bundle exec puma --config example/puma_config.rb example/config.ru

data/example/Gemfile

@@ -1,2 +1,2 @@
 gem 'pubsubstub', path: '../'
-gem 'puma'
+gem 'puma', '~> 3.4'

data/example/config.ru

@@ -1,3 +1,5 @@
+p File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'pubsubstub'
 
 run Pubsubstub::Application

data/example/puma_config.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env puma
+
+# This option is crucial, otherwise puma would wait for all the clients to disconnect before accepting to restart
+# If you run Pubsubstub as a standalone server 0 seconds is recomended since the clients will reconnect
+# and receive the scrollback anyway.
+#
+# But if you run Pubsubstub as an embeded Rack application, you want this to consider what is a
+# decent timeout for the application.
+
+force_shutdown_after 0
+
+
+# min, max thread. The max define how much conccurent clients you can handle.
+# If all threads are occupied the client will be stuck. Make sure to configure this appropriately.
+# If you run Pubsubstub as a standalone, the cost of each thread is really low, so you can easilly have this quite high.
+threads 0, 16
+
+# If you notice your standalone pubsubstub server is starving for CPU, you can run puma in multi-process mode.
+# workers 2

data/lib/pubsubstub.rb

@@ -1,10 +1,15 @@
+require "logger"
+require "mutex_m"
 require "json"
+require "set"
+
 require "sinatra"
-require "em-hiredis"
 require "redis"
 require "pubsubstub/version"
-require "pubsubstub/redis_pub_sub"
+require "pubsubstub/logging"
 require "pubsubstub/channel"
+require "pubsubstub/subscriber"
+require "pubsubstub/subscription"
 require "pubsubstub/event"
 require "pubsubstub/action"
 require "pubsubstub/stream_action"
@@ -12,9 +17,65 @@ require "pubsubstub/publish_action"
 require "pubsubstub/application"
 
 module Pubsubstub
+  extend Mutex_m
+
   class << self
-    attr_accessor :heartbeat_frequency, :redis_url
+    attr_accessor :heartbeat_frequency, :redis_url, :channels_scrollback_size,
+      :channels_scrollback_ttl, :logger, :reconnect_timeout, :error_handler
+
+    def publish(channel_name, *args)
+      Channel.new(channel_name).publish(Event.new(*args))
+      nil
+    end
+
+    def redis_url=(url)
+      @redis_url = url.to_s
+      @redis = nil
+    end
+
+    def redis
+      @redis || synchronize { @redis ||= new_redis }
+    end
+
+    def new_redis
+      Redis.new(url: redis_url)
+    end
+
+    def subscriber
+      @subscriber || synchronize { @subscriber ||= Subscriber.new }
+    end
+
+    def heartbeat_event
+      Event.new('ping', name: 'heartbeat', retry_after: reconnect_timeout)
+    end
+
+    def handle_error(error)
+      logger.error("Uncaught exception: #{error.class}: #{error.message}\n#{error.backtrace.join("\n\t")}\n")
+      error_handler && error_handler.call(error)
+    end
+
+    def report_errors
+      yield
+    rescue => error
+      handle_error(error)
+      raise
+    end
   end
+
+  self.logger = Logger.new(STDOUT)
+  self.logger.level = Logger::DEBUG
   self.heartbeat_frequency = 15
-end
+  self.redis_url = ENV.fetch('REDIS_URL', 'redis://localhost:6379/0')
+  self.channels_scrollback_size = 1000
+  self.channels_scrollback_ttl = 24 * 60 * 60
+  self.reconnect_timeout = 10_000
 
+  # Deprecated. Use Pubsubstub.publish instead
+  module RedisPubSub
+    extend self
+
+    def publish(channel_name, event)
+      Channel.new(channel_name).publish(event)
+    end
+  end
+end

data/lib/pubsubstub/action.rb

@@ -9,15 +9,5 @@ module Pubsubstub
       set :raise_errors, true
       set :show_exceptions, false
     end
-
-    def initialize(*)
-      @channels = Hash.new { |h, k| h[k] = Channel.new(k) }
-      @connections = []
-      super
-    end
-
-    def channel(name)
-      @channels[name]
-    end
   end
 end

data/lib/pubsubstub/channel.rb

@@ -1,54 +1,40 @@
 module Pubsubstub
   class Channel
-    attr_reader :name, :pubsub
-
-    def initialize(name)
-      @name = name
-      @pubsub = RedisPubSub.new(name)
-      @connections = []
-    end
-
-    def subscribe(connection, options = {})
-      listen if @connections.empty?
-      @connections << connection
-      scrollback(connection, options[:last_event_id])
+    class << self
+      def name_from_pubsub_key(key)
+        key.sub(/\.pubsub$/, '')
+      end
     end
 
-    def subscribed?(connection)
-      @connections.include?(connection)
-    end
+    attr_reader :name
 
-    def unsubscribe(connection)
-      @connections.delete(connection)
-      stop_listening if @connections.empty?
+    def initialize(name)
+      @name = name.to_s
     end
 
     def publish(event)
-      pubsub.publish(event)
-    end
-
-    def scrollback(connection, last_event_id)
-      return unless last_event_id
-      pubsub.scrollback(last_event_id) do |event|
-        connection << event.to_message
+      redis.pipelined do
+        redis.zadd(scrollback_key, event.id, event.to_json)
+        redis.zremrangebyrank(scrollback_key, 0, -Pubsubstub.channels_scrollback_size)
+        redis.expire(scrollback_key, Pubsubstub.channels_scrollback_ttl)
+        redis.publish(pubsub_key, event.to_json)
       end
     end
 
-    private
+    def scrollback(since: )
+      redis.zrangebyscore(scrollback_key, Integer(since) + 1, '+inf').map(&Event.method(:from_json))
+    end
 
-    def broadcast(json)
-      string = Event.from_json(json).to_message
-      @connections.each do |connection|
-        connection << string
-      end
+    def scrollback_key
+      "#{name}.scrollback"
     end
 
-    def listen
-      pubsub.subscribe(method(:broadcast))
+    def pubsub_key
+      "#{name}.pubsub"
     end
 
-    def stop_listening
-      pubsub.unsubscribe(method(:broadcast))
+    def redis
+      Pubsubstub.redis
     end
   end
 end

data/lib/pubsubstub/event.rb

@@ -14,12 +14,7 @@ module Pubsubstub
     end
 
     def to_message
-      data = @data.split("\n").map{ |segment| "data: #{segment}" }.join("\n")
-      message = "id: #{id}" << "\n"
-      message << "event: #{name}" << "\n" if name
-      message << "retry: #{retry_after}" << "\n" if retry_after
-      message << data << "\n\n"
-      message
+      @message ||= build_message
     end
 
     def self.from_json(json)
@@ -33,6 +28,15 @@ module Pubsubstub
 
     private
 
+    def build_message
+      data = @data.lines.map{ |segment| "data: #{segment}" }.join
+      message = "id: #{id}\n"
+      message << "event: #{name}\n" if name
+      message << "retry: #{retry_after}\n" if retry_after
+      message << data << "\n\n"
+      message
+    end
+
     def time_now
       (Time.now.to_f * 1000).to_i
     end

data/lib/pubsubstub/logging.rb

@@ -0,0 +1,15 @@
+module Pubsubstub
+  module Logging
+    def error
+      Pubsubstub.logger.error { "[#{self.class.name}] #{yield}" }
+    end
+
+    def info
+      Pubsubstub.logger.info { "[#{self.class.name}] #{yield}" }
+    end
+
+    def debug
+      Pubsubstub.logger.debug { "[#{self.class.name}] #{yield}" }
+    end
+  end
+end

data/lib/pubsubstub/publish_action.rb

@@ -1,9 +1,8 @@
 module Pubsubstub
   class PublishAction < Pubsubstub::Action
     post '/' do
-      event = Event.new(params[:data], name: params[:event])
       (params[:channels] || [:default]).each do |channel_name|
-        channel(channel_name).publish(event)
+        Pubsubstub.publish(channel_name, params[:data], name: params[:event])
       end
       ""
     end

data/lib/pubsubstub/stream_action.rb

@@ -1,10 +1,11 @@
 module Pubsubstub
   class StreamAction < Pubsubstub::Action
-    RECONNECT_TIMEOUT = 10_000
+    include Logging
 
     def initialize(*)
       super
-      start_heartbeat
+      @subscriptions = Set.new
+      @mutex = Mutex.new
     end
 
     get '/', provides: 'text/event-stream' do
@@ -15,71 +16,104 @@ module Pubsubstub
         'Connection' => 'keep-alive',
       })
 
-      if EventMachine.reactor_running?
-        subscribe_connection
+      if event_machine?
+        send_scrollback
       else
-        return_scrollback
+        subscribe_connection
       end
     end
 
+    def call(*)
+      spawn_helper_threads
+      super
+    end
+
     private
 
-    def return_scrollback
-      buffer = ''
-      ensure_connection_has_event(buffer)
+    def last_event_id
+      request.env['HTTP_LAST_EVENT_ID']
+    end
 
-      with_each_channel do |channel|
-        channel.scrollback(buffer, last_event_id)
+    def send_scrollback
+      scrollback_events = []
+      scrollback_events = channels.flat_map { |c| c.scrollback(since: last_event_id) } if last_event_id
+      scrollback_events = [Pubsubstub.heartbeat_event] if scrollback_events.empty?
+      stream do |connection|
+        scrollback_events.each do |event|
+          connection << event.to_message
+        end
       end
+    end
 
-      buffer
+    def event_machine?
+      defined?(EventMachine) && EventMachine.reactor_running?
     end
 
-    def last_event_id
-      request.env['HTTP_LAST_EVENT_ID']
+    def channels
+      (params[:channels] || [:default]).map(&Channel.method(:new))
     end
 
     def subscribe_connection
-      stream(:keep_open) do |connection|
-        @connections << connection
-        ensure_connection_has_event(connection)
-        with_each_channel do |channel|
-          channel.subscribe(connection, last_event_id: last_event_id)
-        end
-
-        connection.callback do
-          @connections.delete(connection)
-          with_each_channel do |channel|
-            channel.unsubscribe(connection)
-          end
+      stream do |connection|
+        subscription = register(channels, connection)
+        begin
+          subscription.stream(last_event_id)
+        ensure
+          release(subscription)
         end
       end
     end
 
-    def ensure_connection_has_event(connection)
-      return if last_event_id
-      connection << heartbeat_event.to_message
+    def register(*args)
+      new_subscription = Subscription.new(*args)
+      @mutex.synchronize { @subscriptions << new_subscription }
+      new_subscription
     end
 
-    def start_heartbeat
-      @heartbeat = Thread.new do
-        loop do
-          sleep Pubsubstub.heartbeat_frequency
-          event = heartbeat_event.to_message
-          @connections.each { |connection| connection << event }
+    def release(subscription)
+      @mutex.synchronize { @subscriptions.delete(subscription) }
+    end
+
+    def spawn_helper_threads
+      return if defined? @helper_threads_initialized
+      @mutex.synchronize do
+        return if defined? @helper_threads_initialized
+        @helper_threads_initialized = true
+        if event_machine?
+          error { "EventMachine is loaded, running in degraded mode :/"}
+        else
+          start_subscriber
+          start_heartbeat
         end
       end
     end
 
-    def with_each_channel(&block)
-      channels = params[:channels] || [:default]
-      channels.each do |channel_name|
-        yield channel(channel_name)
+    def start_subscriber
+      Thread.start do
+        info { "Starting subscriber" }
+        Pubsubstub.report_errors do
+          begin
+            Pubsubstub.subscriber.start
+          rescue Redis::BaseConnectionError => error
+            error { "Can't subscribe to Redis (#{error.class}: #{error.message}). Retrying in 1 second" }
+            sleep 1
+            retry
+          end
+        end
       end
     end
 
-    def heartbeat_event
-      Event.new('ping', name: 'heartbeat', retry_after: RECONNECT_TIMEOUT)
+    def start_heartbeat
+      Thread.start do
+        info { "Starting heartbeat" }
+        Pubsubstub.report_errors do
+          loop do
+            sleep Pubsubstub.heartbeat_frequency
+            event = Pubsubstub.heartbeat_event
+            @subscriptions.each { |subscription| subscription.push(event) }
+          end
+        end
+      end
     end
   end
 end