firehose 1.2.20 → 2.1.0

data/app/assets/javascripts/firehose.js

@@ -0,0 +1,258 @@
+// Firehose: WebSocket client with automatic reconnection and replay
+//
+// Usage (declarative):
+//   <firehose-stream-source path="/firehose" streams="dashboard,user:42"></firehose-stream-source>
+//
+// Usage (imperative):
+//   Firehose.subscribe("/firehose", ["dashboard", "user:42"])
+//   Firehose.unsubscribe("/firehose", ["dashboard"])
+//
+// Each unique path gets its own WebSocket connection.
+// Streams are subscribed/unsubscribed within each connection.
+//
+const Firehose = (function() {
+  // Connection pool: path -> Connection instance
+  const connections = new Map()
+
+  class Connection {
+    constructor(path) {
+      this.path = path
+      this.socket = null
+      this.streams = new Set()
+      this.lastEventId = 0
+      this.reconnectTimer = null
+      this.reconnectDelay = 1000
+      this.intentionalDisconnect = false
+    }
+
+    connect() {
+      // Don't create new connection if socket exists in any state except CLOSED
+      if (this.socket && this.socket.readyState !== WebSocket.CLOSED) {
+        console.debug(`[Firehose:${this.path}] Socket exists, state:`, this.socket.readyState)
+        return
+      }
+
+      this.socket = null
+
+      const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"
+      const url = `${protocol}//${window.location.host}${this.path}`
+
+      console.debug(`[Firehose:${this.path}] Creating connection to`, url)
+      this.intentionalDisconnect = false
+      this.socket = new WebSocket(url)
+
+      this.socket.onopen = () => {
+        console.debug(`[Firehose:${this.path}] Connected`)
+        this.reconnectDelay = 1000
+
+        if (this.streams.size > 0) {
+          console.debug(`[Firehose:${this.path}] Subscribing to`, Array.from(this.streams), "from event", this.lastEventId)
+          this.send({
+            command: "subscribe",
+            streams: Array.from(this.streams),
+            last_event_id: this.lastEventId
+          })
+        }
+      }
+
+      this.socket.onmessage = (event) => {
+        const data = JSON.parse(event.data)
+        this.handleEvent(data)
+      }
+
+      this.socket.onclose = () => {
+        console.debug(`[Firehose:${this.path}] Closed, intentional:`, this.intentionalDisconnect)
+        this.socket = null
+
+        if (!this.intentionalDisconnect && this.streams.size > 0) {
+          this.scheduleReconnect()
+        } else if (this.streams.size === 0) {
+          // Remove from pool if no streams
+          connections.delete(this.path)
+        }
+      }
+
+      this.socket.onerror = (error) => {
+        console.debug(`[Firehose:${this.path}] Error`, error)
+      }
+    }
+
+    disconnect() {
+      console.debug(`[Firehose:${this.path}] Disconnect called`)
+      clearTimeout(this.reconnectTimer)
+      this.intentionalDisconnect = true
+      if (this.socket) {
+        this.socket.close()
+      }
+    }
+
+    scheduleReconnect() {
+      clearTimeout(this.reconnectTimer)
+      if (this.streams.size === 0) return
+      console.debug(`[Firehose:${this.path}] Reconnecting in`, this.reconnectDelay, "ms")
+      this.reconnectTimer = setTimeout(() => {
+        this.connect()
+        this.reconnectDelay = Math.min(this.reconnectDelay * 2, 30000)
+      }, this.reconnectDelay)
+    }
+
+    send(message) {
+      if (this.socket && this.socket.readyState === WebSocket.OPEN) {
+        this.socket.send(JSON.stringify(message))
+      }
+    }
+
+    subscribe(streamList) {
+      const newStreams = streamList.filter(s => !this.streams.has(s))
+      console.debug(`[Firehose:${this.path}] Subscribe:`, streamList, "new:", newStreams)
+
+      if (newStreams.length === 0) return
+
+      newStreams.forEach(s => this.streams.add(s))
+
+      if (this.socket && this.socket.readyState === WebSocket.OPEN) {
+        this.send({
+          command: "subscribe",
+          streams: newStreams,
+          last_event_id: this.lastEventId
+        })
+      } else if (!this.socket || this.socket.readyState === WebSocket.CLOSED) {
+        this.connect()
+      } else if (this.socket.readyState === WebSocket.CLOSING) {
+        // Socket is closing - ensure onclose will reconnect
+        console.debug(`[Firehose:${this.path}] Socket closing, will reconnect`)
+        this.intentionalDisconnect = false
+      }
+    }
+
+    unsubscribe(streamList) {
+      const toRemove = streamList.filter(s => this.streams.has(s))
+      console.debug(`[Firehose:${this.path}] Unsubscribe:`, streamList, "removing:", toRemove)
+
+      if (toRemove.length === 0) return
+
+      toRemove.forEach(s => this.streams.delete(s))
+
+      if (this.socket && this.socket.readyState === WebSocket.OPEN) {
+        this.send({
+          command: "unsubscribe",
+          streams: toRemove
+        })
+      }
+
+      if (this.streams.size === 0) {
+        this.disconnect()
+      }
+    }
+
+    handleEvent(event) {
+      console.debug(`[Firehose:${this.path}] Event:`, event)
+
+      if (event.id) {
+        this.lastEventId = Math.max(this.lastEventId, event.id)
+      }
+
+      const customEvent = new CustomEvent("firehose:message", {
+        bubbles: true,
+        detail: { ...event, path: this.path }
+      })
+      document.dispatchEvent(customEvent)
+
+      if (event.data === "refresh") {
+        refresh()
+      }
+    }
+  }
+
+  function getConnection(path) {
+    if (!connections.has(path)) {
+      connections.set(path, new Connection(path))
+    }
+    return connections.get(path)
+  }
+
+  function subscribe(path, streamList) {
+    getConnection(path).subscribe(streamList)
+  }
+
+  function unsubscribe(path, streamList) {
+    const conn = connections.get(path)
+    if (conn) {
+      conn.unsubscribe(streamList)
+    }
+  }
+
+  function refresh() {
+    console.debug("[Firehose] Triggering page refresh")
+    if (typeof Turbo !== "undefined") {
+      if (Turbo.session && typeof Turbo.session.refresh === "function") {
+        Turbo.session.refresh(document.baseURI)
+      } else {
+        Turbo.visit(window.location.href, { action: "replace" })
+      }
+    } else {
+      window.location.reload()
+    }
+  }
+
+  return {
+    subscribe,
+    unsubscribe,
+    getConnection,
+    get connections() { return connections }
+  }
+})()
+
+// Custom element for declarative stream subscriptions
+class FirehoseStreamSource extends HTMLElement {
+  connectedCallback() {
+    this._path = this.getAttribute("path") || "/firehose"
+    const streams = this.getAttribute("streams")
+    console.debug("[Firehose] Element connected, path:", this._path, "streams:", streams)
+
+    if (streams) {
+      this._streams = streams.split(",").map(s => s.trim()).filter(Boolean)
+      Firehose.subscribe(this._path, this._streams)
+    }
+  }
+
+  disconnectedCallback() {
+    console.debug("[Firehose] Element disconnected, path:", this._path, "streams:", this._streams)
+    if (this._streams && this._path) {
+      Firehose.unsubscribe(this._path, this._streams)
+    }
+  }
+
+  static get observedAttributes() {
+    return ["streams", "path"]
+  }
+
+  attributeChangedCallback(name, oldValue, newValue) {
+    // Only handle changes after initial connection
+    if (oldValue === null) return
+
+    if (name === "streams") {
+      const oldStreams = oldValue.split(",").map(s => s.trim()).filter(Boolean)
+      const newStreams = newValue.split(",").map(s => s.trim()).filter(Boolean)
+
+      const toRemove = oldStreams.filter(s => !newStreams.includes(s))
+      const toAdd = newStreams.filter(s => !oldStreams.includes(s))
+
+      if (toRemove.length) Firehose.unsubscribe(this._path, toRemove)
+      if (toAdd.length) Firehose.subscribe(this._path, toAdd)
+
+      this._streams = newStreams
+    }
+  }
+}
+
+if (!customElements.get("firehose-stream-source")) {
+  customElements.define("firehose-stream-source", FirehoseStreamSource)
+}
+
+// Export for module systems
+if (typeof module !== "undefined" && module.exports) {
+  module.exports = Firehose
+} else if (typeof window !== "undefined") {
+  window.Firehose = Firehose
+}

data/config/importmap.rb

@@ -0,0 +1 @@
+pin "firehose", to: "firehose.js"

data/db/migrate/20260214185244_create_firehose_events.rb

@@ -0,0 +1,23 @@
+class CreateFirehoseEvents < ActiveRecord::Migration[8.1]
+  def change
+    create_table :firehose_channels do |t|
+      t.text :name, null: false
+      t.integer :sequence, null: false, default: 0
+      t.integer :messages_count, null: false, default: 0
+      t.timestamps
+    end
+
+    add_index :firehose_channels, :name, unique: true
+
+    create_table :firehose_messages do |t|
+      t.references :channel,
+        null: false,
+        foreign_key: { to_table: :firehose_channels, on_delete: :cascade }
+      t.integer :sequence, null: false
+      t.text :data, null: false
+      t.datetime :created_at, null: false, default: -> { "now()" }
+    end
+
+    add_index :firehose_messages, [:channel_id, :sequence], unique: true
+  end
+end

data/lib/commands/firehose_command.rb

@@ -0,0 +1,119 @@
+module Firehose
+  module Command
+    class FirehoseCommand < Rails::Command::Base
+      namespace "firehose"
+
+      desc "status", "Show Firehose stats (--watch to refresh)"
+      option :watch, type: :numeric, default: nil, aliases: "-w", lazy_default: 1,
+        desc: "Refresh every N seconds (default: 1)"
+      def status
+        boot_application!
+
+        if options[:watch]
+          trap("INT") { puts; exit }
+          loop do
+            print "\e[2J\e[H"
+            print_stats
+            sleep options[:watch]
+          end
+        else
+          print_stats
+        end
+      end
+
+      default_command :status
+
+      desc "channels", "List Firehose channels"
+      option :limit, type: :numeric, default: 50, aliases: "-n",
+        desc: "Number of channels to show"
+      def channels
+        boot_application!
+
+        total = Models::Channel.count
+        channels = Models::Channel.order(sequence: :desc).limit(options[:limit])
+
+        if channels.empty?
+          puts "No channels."
+          return
+        end
+
+        name_width = [channels.map { |c| c.name.length }.max, 40].min
+
+        puts "%-#{name_width}s  %8s  %8s  %s" % ["NAME", "SEQUENCE", "MESSAGES", "LATEST MESSAGE"]
+        puts "-" * (name_width + 50)
+
+        channels.each do |ch|
+          latest = ch.messages.order(id: :desc).limit(1).pick(:created_at)
+          latest_at = latest ? latest.strftime("%Y-%m-%d %H:%M:%S") : "-"
+          puts "%-#{name_width}s  %8d  %8d  %s" % [ch.name.truncate(name_width), ch.sequence, ch.messages_count, latest_at]
+        end
+
+        puts
+        puts "#{channels.size} of #{total} channel(s)"
+      end
+
+      desc "messages", "List recent messages (use --tail to follow live)"
+      option :tail, type: :boolean, default: false, aliases: "-f",
+        desc: "Follow new messages as they arrive"
+      option :channel, type: :string, aliases: "-c",
+        desc: "Filter to a specific channel"
+      option :limit, type: :numeric, default: 20, aliases: "-n",
+        desc: "Number of recent messages to show"
+      def messages
+        boot_application!
+
+        scope = Models::Message.includes(:channel).order(id: :desc)
+        scope = scope.joins(:channel).where(firehose_channels: { name: options[:channel] }) if options[:channel]
+
+        messages = scope.limit(options[:limit]).to_a.reverse
+
+        if messages.any?
+          messages.each { |msg| print_message(msg) }
+        else
+          puts "No messages."
+        end
+
+        return unless options[:tail]
+
+        trap("INT") { puts; exit }
+
+        last_id = Models::Message.maximum(:id) || 0
+
+        loop do
+          new_scope = Models::Message.where("id > ?", last_id).includes(:channel).order(:id).limit(100)
+          new_scope = new_scope.joins(:channel).where(firehose_channels: { name: options[:channel] }) if options[:channel]
+
+          new_scope.each do |msg|
+            print_message(msg)
+            last_id = msg.id
+          end
+
+          sleep 0.5
+        end
+      end
+
+      desc "install", "Install Firehose (creates controller, routes, JS import, runs migrations)"
+      def install
+        boot_application!
+        Rails::Generators.invoke("firehose:install", [], behavior: :invoke)
+      end
+
+      private
+
+      def print_stats
+        puts "channels: #{Models::Channel.count}"
+        puts "messages:"
+        puts "  total: #{Models::Message.count}"
+        puts "  last_minute: #{Models::Message.where("created_at > ?", 1.minute.ago).count}"
+        puts "  last_hour: #{Models::Message.where("created_at > ?", 1.hour.ago).count}"
+        puts "  last_day: #{Models::Message.where("created_at > ?", 1.day.ago).count}"
+      end
+
+      def print_message(msg)
+        time = msg.created_at.strftime("%H:%M:%S.%L")
+        data = msg.data.truncate(80)
+        puts "[#{time}] ##{msg.id} #{msg.channel.name} seq:#{msg.sequence} #{data}"
+      end
+    end
+  end
+end

data/lib/firehose/channel.rb

@@ -0,0 +1,40 @@
+module Firehose
+  # Public API for interacting with a named stream.
+  #
+  #   channel = Firehose.channel("dashboard")
+  #   channel.publish("refresh")
+  #
+  #   sub = channel.subscribe { |payload| puts payload }
+  #   sub.close
+  #
+  class Channel
+    attr_reader :name
+
+    def initialize(name, server: Firehose.server)
+      @name = name.to_s
+      @server = server
+    end
+
+    def publish(data)
+      @server.broadcast(@name, data)
+    end
+
+    def subscribe(&callback)
+      pg_channel = @server.channel_name(@name)
+      @server.subscribe(pg_channel, callback)
+      Subscription.new(@server, pg_channel, callback)
+    end
+
+    class Subscription
+      def initialize(server, channel, callback)
+        @server = server
+        @channel = channel
+        @callback = callback
+      end
+
+      def close
+        @server.unsubscribe(@channel, @callback)
+      end
+    end
+  end
+end

data/lib/firehose/cleanup_job.rb

@@ -0,0 +1,21 @@
+module Firehose
+  class CleanupJob < ActiveJob::Base
+    def perform(stream)
+      channel = Models::Channel.find_by(name: stream)
+      return unless channel
+
+      cutoff_id = channel.messages
+        .order(sequence: :desc)
+        .offset(Firehose.server.cleanup_threshold)
+        .limit(1)
+        .pick(:id)
+
+      return unless cutoff_id
+
+      channel.messages.where("id <= ?", cutoff_id).delete_all
+      channel.update_columns(messages_count: channel.messages.count)
+
+      channel.destroy if channel.messages_count == 0
+    end
+  end
+end

data/lib/firehose/engine.rb

@@ -0,0 +1,36 @@
+module Firehose
+  class Engine < ::Rails::Engine
+    initializer "firehose.assets" do |app|
+      if app.config.respond_to?(:assets)
+        app.config.assets.paths << root.join("app/assets/javascripts")
+      end
+
+      if defined?(Importmap)
+        app.config.importmap.paths << root.join("config/importmap.rb")
+      end
+    end
+
+    initializer "firehose.migrations" do |app|
+      app.config.paths["db/migrate"].concat(config.paths["db/migrate"].expanded)
+    end
+
+    initializer "firehose.server" do |app|
+      Firehose.server = Firehose::Server.new
+
+      rb_config = app.root.join("config/firehose.rb")
+      yml_config = app.root.join("config/firehose.yml")
+
+      if rb_config.exist?
+        load rb_config
+      elsif yml_config.exist?
+        Firehose.server.configure_from_yaml(yml_config)
+      end
+
+      Firehose.server.start
+    end
+
+    at_exit do
+      Firehose.server.shutdown rescue nil
+    end
+  end
+end

data/lib/firehose/helper.rb

@@ -0,0 +1,40 @@
+module Firehose
+  module Helper
+    # Renders a <firehose-stream-source> element for the given streamables.
+    #
+    # Usage in Phlex:
+    #   firehose_stream_from @report
+    #   firehose_stream_from "dashboard", "user:#{current_user.id}"
+    #   firehose_stream_from @report, path: "/my-cable"
+    #
+    def firehose_stream_from(*streamables, path: "/firehose")
+      streams = streamables.map { |s| stream_name_for(s) }.join(",")
+      render FirehoseStreamSource.new(streams:, path:)
+    end
+
+    private
+
+    def stream_name_for(streamable)
+      case streamable
+      when String, Symbol
+        streamable.to_s
+      else
+        # ActiveRecord model - use global ID or model name + id
+        streamable.to_gid_param
+      end
+    end
+
+    class FirehoseStreamSource < Phlex::HTML
+      register_element :firehose_stream_source
+
+      def initialize(streams:, path:)
+        @streams = streams
+        @path = path
+      end
+
+      def view_template
+        firehose_stream_source(streams: @streams, path: @path)
+      end
+    end
+  end
+end

data/lib/firehose/models/channel.rb

@@ -0,0 +1,12 @@
+module Firehose
+  module Models
+    class Channel < ActiveRecord::Base
+      self.table_name = "firehose_channels"
+
+      has_many :messages,
+        class_name: "Firehose::Models::Message",
+        foreign_key: :channel_id,
+        dependent: :delete_all
+    end
+  end
+end

data/lib/firehose/models/message.rb

@@ -0,0 +1,11 @@
+module Firehose
+  module Models
+    class Message < ActiveRecord::Base
+      self.table_name = "firehose_messages"
+
+      belongs_to :channel,
+        class_name: "Firehose::Models::Channel",
+        counter_cache: :messages_count
+    end
+  end
+end

data/lib/firehose/queue.rb

@@ -0,0 +1,44 @@
+module Firehose
+  # Raw push/pop queue over PG LISTEN/NOTIFY.
+  #
+  # For ephemeral signaling (auth nonces, job completion, etc.) where
+  # messages don't need database persistence or replay.
+  #
+  # For database-backed stream consumption with replay and ordering,
+  # use Server::Subscriptions with the SSE or WebSocket transports.
+  class Queue
+    class TimeoutError < StandardError; end
+
+    def initialize(channel, server: Firehose.server)
+      @channel = channel
+      @server = server
+      @queue = ::Queue.new
+      @callback = ->(payload) { @queue << payload }
+      @subscribed = false
+    end
+
+    def subscribe
+      return self if @subscribed
+      @subscribed = true
+      @server.subscribe(@channel, @callback)
+      self
+    end
+
+    def push(message)
+      @server.notify(@channel, message)
+    end
+
+    def pop(timeout: nil)
+      subscribe
+      payload = @queue.pop(timeout:)
+      raise TimeoutError, "No message received on #{@channel} within #{timeout}s" if payload.nil? && timeout
+      payload
+    end
+
+    def close
+      @server.unsubscribe(@channel, @callback) if @subscribed
+      @subscribed = false
+      @queue.close
+    end
+  end
+end