firehose 1.2.20 → 2.1.0

data/lib/firehose/sse.rb

@@ -0,0 +1,187 @@
+require "async/queue"
+
+module Firehose
+  # SSE (Server-Sent Events) transport for Firehose.
+  #
+  # Usage:
+  #   class CableController < ApplicationController
+  #     include Firehose::SSE
+  #
+  #     before_action :authenticate_user!
+  #
+  #     def authorize_streams(streams)
+  #       streams.select { |s| current_user.can_access?(s) }
+  #     end
+  #   end
+  #
+  # Route:
+  #   get "cable/sse", to: "cable#sse"
+  #
+  # Client:
+  #   const source = new EventSource("/cable/sse?streams=foo,bar")
+  #   source.addEventListener("foo", (e) => console.log(e.data))
+  #
+  # Replay:
+  #   Browser automatically sends Last-Event-ID header on reconnect.
+  #
+  module SSE
+    extend ActiveSupport::Concern
+    include Streamable
+
+    # Seconds between SSE comment-line keepalives. Dead clients surface
+    # via the write failing; intermediaries that time out idle HTTP
+    # connections are kept happy by the steady trickle.
+    KEEPALIVE_INTERVAL = 30
+
+    def sse
+      streams = parse_sse_streams
+      streams = authorize_streams(streams)
+
+      if streams.empty?
+        head :forbidden
+        return
+      end
+
+      response.headers["Content-Type"] = "text/event-stream"
+      response.headers["Cache-Control"] = "no-cache, no-store"
+      response.headers["X-Accel-Buffering"] = "no"
+      response.headers["Connection"] = "keep-alive"
+
+      handler = SSEHandler.new(request, response, streams:, controller: self)
+      handler.run
+    end
+
+    private
+
+    def parse_sse_streams
+      streams_param = params[:streams]
+      return [] unless streams_param
+
+      streams_param.split(",").map(&:strip).reject(&:empty?)
+    end
+
+    class SSEHandler
+      def initialize(request, response, streams:, controller:)
+        @request = request
+        @response = response
+        @streams = streams
+        @controller = controller
+        @subscriptions = Firehose.server.subscriptions
+        @queue = Async::Queue.new
+      end
+
+      def run
+        task = Async::Task.current
+
+        writer_task = task.async do
+          write_messages
+        end
+
+        replay_events
+        subscribe_to_streams
+        writer_task.wait
+      rescue Protocol::HTTP::Error, EOFError, Async::Stop, IOError, Errno::EPIPE
+        # Client disconnected
+      ensure
+        writer_task&.stop
+        @subscriptions.close
+        @queue.close
+      end
+
+      private
+
+      def last_event_id
+        @request.headers["Last-Event-ID"].to_i
+      end
+
+      # See WebSocketHandler#replay_events — same gap-detection logic
+      # for SSE. On replay_gap the client sees an event with the magic
+      # "firehose:replay_gap" event name plus details; EventSource
+      # dispatches it like any custom event.
+      def replay_events
+        return unless last_event_id > 0
+
+        channels = Models::Channel.where(name: @streams).to_a
+        channels.each do |ch|
+          oldest = ch.messages.minimum(:id)
+          next unless oldest
+
+          if oldest > last_event_id
+            current = ch.messages.maximum(:id)
+            write_event(
+              error: "replay_gap",
+              stream: ch.name,
+              last_event_id: last_event_id,
+              oldest_retained_id: oldest,
+              current_id: current
+            )
+            next
+          end
+
+          ch.messages
+            .where("id > ?", last_event_id)
+            .order(:id)
+            .find_each { |msg| write_event(id: msg.id, channel_id: msg.channel_id, sequence: msg.sequence, stream: ch.name, data: msg.data) }
+        end
+      end
+
+      def subscribe_to_streams
+        @streams.each do |stream|
+          @subscriptions.add(stream) { |payload| @queue.enqueue(payload) }
+        end
+
+        # Keep connection open waiting for events
+        sleep
+      rescue Async::Stop
+        # Task stopped
+      end
+
+      def write_messages
+        loop do
+          payload = @queue.dequeue(timeout: KEEPALIVE_INTERVAL)
+
+          if payload.nil?
+            # Idle — write an SSE comment line. Clients ignore it; dead
+            # clients surface via write failure propagating as IOError.
+            @response.body.write(": keepalive\n\n")
+            next
+          end
+
+          event = JSON.parse(payload, symbolize_names: true)
+          event = resolve_event(event) unless event.key?(:data)
+          write_event(event) if event
+        end
+      rescue Async::Stop
+        # Task stopped
+      end
+
+      def resolve_event(event)
+        msg = Models::Message.includes(:channel).find_by(id: event[:id])
+        return unless msg
+        { id: msg.id, channel_id: msg.channel_id, sequence: msg.sequence, stream: msg.channel.name, data: msg.data }
+      end
+
+      def write_event(event)
+        event = event.transform_keys(&:to_sym)
+        event = @controller.build_event(event)
+        return unless event
+
+        body = @response.body
+
+        if event[:error]
+          # Error events (e.g., replay_gap) use a dedicated event name
+          # so clients can register targeted listeners:
+          #   eventSource.addEventListener("firehose_replay_gap", handler)
+          body.write("event: firehose_#{event[:error]}\n")
+          body.write("data: #{event.except(:error).to_json}\n\n")
+          return
+        end
+
+        body.write("id: #{event[:id]}\n")
+        body.write("event: #{event[:stream]}\n")
+        data = { data: event[:data], channel_id: event[:channel_id], sequence: event[:sequence] }.to_json
+        body.write("data: #{data}\n\n")
+      end
+    end
+  end
+end

data/lib/firehose/stream.rb

@@ -0,0 +1,32 @@
+module Firehose
+  # Include both WebSocket and SSE transports.
+  #
+  # Usage:
+  #   class CableController < ApplicationController
+  #     include Firehose::Stream
+  #
+  #     before_action :authenticate_user!
+  #
+  #     def authorize_streams(streams)
+  #       streams.select { |s| current_user.can_access?(s) }
+  #     end
+  #
+  #     def build_event(event)
+  #       event  # or transform/filter
+  #     end
+  #   end
+  #
+  # Routes:
+  #   match "cable", to: "cable#websocket", via: [:get, :connect]
+  #   get "cable/sse", to: "cable#sse"
+  #
+  # Or include just one transport:
+  #   include Firehose::WebSocket
+  #   include Firehose::SSE
+  #
+  module Stream
+    extend ActiveSupport::Concern
+    include WebSocket
+    include SSE
+  end
+end

data/lib/firehose/streamable.rb

@@ -0,0 +1,25 @@
+# Shared base for WebSocket and SSE transports.
+# Include this module to get the callback interface without a transport.
+#
+# Callbacks (override in controller):
+#   authorize_streams(streams) - Filter which streams user can subscribe to
+#   build_event(event)         - Transform event hash before sending, or nil to skip
+#
+module Firehose
+  module Streamable
+    extend ActiveSupport::Concern
+
+    # Override to authorize stream access.
+    # Return only the streams the user is allowed to subscribe to.
+    def authorize_streams(streams)
+      streams
+    end
+
+    # Override to transform events before sending.
+    # Return nil to skip sending the event.
+    # Event is a hash: { id:, stream:, data: }
+    def build_event(event)
+      event
+    end
+  end
+end

data/lib/firehose/websocket.rb

@@ -0,0 +1,242 @@
+require "async/websocket/adapters/rails"
+require "async/queue"
+
+module Firehose
+  # WebSocket transport for Firehose.
+  #
+  # Usage:
+  #   class CableController < ApplicationController
+  #     include Firehose::WebSocket
+  #
+  #     before_action :authenticate_user!
+  #
+  #     def authorize_streams(streams)
+  #       streams.select { |s| current_user.can_access?(s) }
+  #     end
+  #   end
+  #
+  # Route:
+  #   match "cable", to: "cable#websocket", via: [:get, :connect]
+  #
+  # Protocol:
+  #   Client sends:   { "command": "subscribe", "streams": ["a", "b"], "last_event_id": 123 }
+  #   Client sends:   { "command": "unsubscribe", "streams": ["a"] }
+  #   Server sends:   { "id": 456, "stream": "a", "data": "refresh" }
+  #
+  module WebSocket
+    extend ActiveSupport::Concern
+    include Streamable
+
+    # Seconds between server-initiated WS ping frames when the stream is
+    # idle. Browsers auto-respond with pong frames at the protocol level
+    # (RFC 6455), so no client code is needed.
+    PING_INTERVAL = 30
+
+    # Deadline after which a lack of pong is treated as a dead client.
+    # Two missed pings — generous enough to tolerate jittery networks,
+    # tight enough to reclaim leaked half-open connections promptly.
+    PONG_TIMEOUT = 90
+
+    # Upper bound on per-connection outgoing queue depth. A healthy client
+    # drains events as fast as the server produces them; if the queue
+    # fills, the client is too slow (bad network, backgrounded tab that
+    # stopped reading). We disconnect rather than buffering without bound.
+    # Clients reconnect and receive replay_gap if their cursor is stale.
+    QUEUE_LIMIT = 1000
+
+    def websocket
+      self.response = Async::WebSocket::Adapters::Rails.open(request) do |connection|
+        handler = WebSocketHandler.new(connection, controller: self)
+        handler.run
+      end
+    end
+
+    class WebSocketHandler
+      def initialize(connection, controller:)
+        @connection = connection
+        @controller = controller
+        @streams = Set.new
+        @subscriptions = Firehose.server.subscriptions
+        @queue = Async::Queue.new
+        @queue_limit = QUEUE_LIMIT
+        @overflow = false
+        @last_pong_at = monotonic_now
+        install_pong_tracker
+      end
+
+      def run
+        task = Async::Task.current
+
+        writer_task = task.async do
+          write_messages
+        end
+
+        read_messages
+        writer_task.stop
+      rescue Protocol::WebSocket::ClosedError, EOFError, Async::Stop
+        # Client disconnected
+      ensure
+        @subscriptions.close
+        @streams.clear
+        @queue.close
+      end
+
+      private
+
+      # Override the connection's pong handler so we can track liveness.
+      # Pings are sent from write_messages below; pongs arrive via the
+      # read fiber, which calls into the connection's receive_pong.
+      def install_pong_tracker
+        tracker = self
+        @connection.define_singleton_method(:receive_pong) do |_frame|
+          tracker.send(:record_pong)
+        end
+      end
+
+      def record_pong
+        @last_pong_at = monotonic_now
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def read_messages
+        while (message = @connection.read)
+          handle_message(JSON.parse(message.to_str))
+        end
+      rescue JSON::ParserError
+        # Ignore malformed messages
+      end
+
+      # Owns the only write path to the connection: event frames AND
+      # heartbeat pings. Piggybacking pings here avoids concurrent-write
+      # races with a separate heartbeat fiber.
+      #
+      # When idle (dequeue times out), send a ping if we're overdue.
+      # If no pong has been seen within PONG_TIMEOUT, close the connection
+      # — read_messages will exit cleanly via ClosedError.
+      def write_messages
+        loop do
+          payload = @queue.dequeue(timeout: PING_INTERVAL)
+
+          if payload.nil?
+            if monotonic_now - @last_pong_at > PONG_TIMEOUT
+              Firehose.logger.warn {
+                "[Firehose] WebSocket pong timeout (> #{PONG_TIMEOUT}s), closing connection"
+              }
+              @connection.close
+              return
+            end
+            @connection.send_ping
+            next
+          end
+
+          event = JSON.parse(payload, symbolize_names: true)
+          event = resolve_event(event) unless event.key?(:data)
+          send_event(event) if event
+        end
+      rescue Async::Stop
+        # Task stopped
+      end
+
+      def resolve_event(event)
+        msg = Models::Message.includes(:channel).find_by(id: event[:id])
+        return unless msg
+        { id: msg.id, channel_id: msg.channel_id, sequence: msg.sequence, stream: msg.channel.name, data: msg.data }
+      end
+
+      def handle_message(msg)
+        case msg["command"]
+        when "subscribe"
+          subscribe(msg["streams"] || [], msg["last_event_id"].to_i)
+        when "unsubscribe"
+          unsubscribe(msg["streams"] || [])
+        end
+      end
+
+      def subscribe(streams, last_event_id)
+        streams = Array(streams).map(&:to_s)
+        streams = @controller.authorize_streams(streams)
+
+        new_streams = streams - @streams.to_a
+        @streams.merge(new_streams)
+
+        replay_events(new_streams, last_event_id) if new_streams.any? && last_event_id > 0
+
+        new_streams.each do |stream|
+          @subscriptions.add(stream) { |payload| enqueue_or_disconnect(payload) }
+        end
+      end
+
+      # Called from the Firehose consumer thread — must return quickly so
+      # we don't block other subscribers. If our queue is full, the client
+      # is draining too slowly: close the connection (cleanup runs via the
+      # rescue in #run) and stop accepting more events. The client will
+      # reconnect and hit replay_gap if its cursor has aged past retention.
+      def enqueue_or_disconnect(payload)
+        if @queue.size >= @queue_limit
+          return if @overflow
+          @overflow = true
+          Firehose.logger.warn {
+            "[Firehose] WS queue depth #{@queue.size} >= #{@queue_limit}; disconnecting slow client"
+          }
+          begin
+            @connection.close
+          rescue => e
+            Firehose.logger.debug { "[Firehose] Error closing overflow client: #{e.message}" }
+          end
+        else
+          @queue.enqueue(payload)
+        end
+      end
+
+      def unsubscribe(streams)
+        Array(streams).map(&:to_s).each do |stream|
+          @streams.delete(stream)
+          @subscriptions.remove(stream)
+        end
+      end
+
+      # Replay messages the client missed, per stream. If a stream's
+      # oldest retained message is newer than last_event_id, the gap
+      # exceeds our retention window — emit a replay_gap error for that
+      # stream so the client can decide how to recover (reload, API
+      # refetch, surface to user). Streams with an intact window get
+      # normal replay.
+      def replay_events(streams, since_id)
+        channels = Models::Channel.where(name: streams).to_a
+        channels.each do |ch|
+          oldest = ch.messages.minimum(:id)
+          next unless oldest  # no messages at all — nothing to replay
+
+          if oldest > since_id
+            current = ch.messages.maximum(:id)
+            send_event(
+              error: "replay_gap",
+              stream: ch.name,
+              last_event_id: since_id,
+              oldest_retained_id: oldest,
+              current_id: current
+            )
+            next
+          end
+
+          ch.messages
+            .where("id > ?", since_id)
+            .order(:id)
+            .find_each { |msg| send_event(id: msg.id, channel_id: msg.channel_id, sequence: msg.sequence, stream: ch.name, data: msg.data) }
+        end
+      end
+
+      def send_event(event)
+        event = @controller.build_event(event.transform_keys(&:to_sym))
+        return unless event
+
+        message = Protocol::WebSocket::TextMessage.generate(event)
+        @connection.write(message)
+        @connection.flush
+      end
+    end
+  end
+end

data/lib/firehose.rb

@@ -1,19 +1,71 @@
-ENV['RACK_ENV'] ||= 'development' # TODO - Lets not rock out envs like its 1999.
+# Firehose: WebSocket and SSE streaming with database-backed replay
+#
+# Usage:
+#   Firehose.channel("dashboard").publish("refresh")
+#
+#   sub = Firehose.channel("dashboard").subscribe { |event| puts event }
+#   sub.close
+#
+# Controller:
+#   class FirehoseController < ApplicationController
+#     include Firehose::Stream
+#   end
+#
+module Firehose
+  VERSION = "2.1.0"
 
-require 'uri'
-require 'firehose/version'
-require 'firehose/logging'
+  module Models
+    autoload :Channel, "firehose/models/channel"
+    autoload :Message, "firehose/models/message"
+  end
 
-# TODO - Figure if we need to have an if/else for Rails::Engine loading and Firehose::Assets::Sprockets.auto_detect
-require 'firehose/rails' if defined?(::Rails::Engine)
+  class << self
+    def channel(name)
+      Channel.new(name, server:)
+    end
 
-module Firehose
-  autoload :Server,       'firehose/server'
-  autoload :Client,       'firehose/client'
-  autoload :Assets,       'firehose/assets'
-  autoload :Rack,         'firehose/rack'
-  autoload :CLI,          'firehose/cli'
-
-  # Default URI for the Firehose server. Consider the port "well-known" and bindable from other apps.
-  URI = URI.parse("//0.0.0.0:7474").freeze
+    def server
+      @server || raise("Firehose::Server not started. Ensure the Firehose engine has initialized.")
+    end
+
+    def server=(server)
+      @server = server
+    end
+
+    def logger
+      @logger ||= defined?(Rails) ? Rails.logger : Logger.new($stdout, progname: "Firehose")
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+
+    # Optional metrics hook. Called periodically from a supervised thread
+    # with a flat hash of current gauges and counters:
+    #
+    #   Firehose.on_metrics = ->(metrics) {
+    #     metrics.each { |k, v| StatsD.gauge("firehose.#{k}", v) }
+    #   }
+    #
+    # Set to nil (default) to disable emission. Gem never raises from a
+    # metrics call — if your reporter throws, the error is swallowed.
+    def on_metrics
+      @on_metrics
+    end
+
+    def on_metrics=(callable)
+      @on_metrics = callable
+    end
+  end
 end
+
+require_relative "firehose/engine"
+require_relative "firehose/channel"
+require_relative "firehose/server"
+require_relative "firehose/cleanup_job"
+require_relative "firehose/streamable"
+require_relative "firehose/websocket"
+require_relative "firehose/sse"
+require_relative "firehose/queue"
+require_relative "firehose/stream"
+require_relative "firehose/helper" if defined?(Phlex)

data/lib/generators/firehose/install/install_generator.rb

@@ -0,0 +1,43 @@
+module Firehose
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      def create_controller
+        template "firehose_controller.rb", "app/controllers/firehose_controller.rb"
+      end
+
+      def add_routes
+        route <<~RUBY
+          match "firehose", to: "firehose#websocket", via: [:get, :connect]
+          get "firehose/sse", to: "firehose#sse"
+        RUBY
+      end
+
+      def add_javascript_import
+        js_file = "app/javascript/application.js"
+        return unless File.exist?(js_file)
+
+        import_line = 'import "firehose"'
+        return if File.read(js_file).include?(import_line)
+
+        append_to_file js_file, "\n#{import_line}\n"
+      end
+
+      def run_migrations
+        rails_command "db:migrate"
+      end
+
+      def print_next_steps
+        say ""
+        say "Firehose installed!", :green
+        say ""
+        say "  Broadcast:   Firehose.channel(\"my-stream\").publish(\"refresh\")"
+        say "  Subscribe:   <firehose-stream-source streams=\"my-stream\"></firehose-stream-source>"
+        say ""
+        say "  Edit app/controllers/firehose_controller.rb to add authentication."
+        say ""
+      end
+    end
+  end
+end

data/lib/generators/firehose/install/templates/firehose_controller.rb

@@ -0,0 +1,16 @@
+class FirehoseController < ApplicationController
+  include Firehose::Stream
+
+  # Public endpoint - no auth required.
+  # Uncomment below to add authentication and stream authorization.
+
+  # before_action :authenticate_user!
+  #
+  # def authorize_streams(streams)
+  #   streams.select { |s| current_user.can_access?(s) }
+  # end
+  #
+  # def build_event(event)
+  #   event  # transform or return nil to skip
+  # end
+end