ration 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 16eee91f9339a1e635c1913995515056c4e154bc1b8e5a657917591e0aaf033d
+  data.tar.gz: 4f059b3b5a90c4c8dcd72075ba231488248c391aebb528a916eef511e33ce20b
+SHA512:
+  metadata.gz: 3be362124e709b25d33df4a9f7997aac2c592c689c6c98d7e27fb825da626af4749dccaf3c3f15244ff50d9e6a1aa5cc181c6553d74f5f333aacf88f8396d42a
+  data.tar.gz: f467fa48b7727e23a3b9f799ac218aea86589da4270a8f60e2cb0506c0b739f88718fa9f7671699aeac455d4ac0c0de43376e67373e3c27b3eab756f47bfacdb

data/lib/ration/backends/base.rb

@@ -0,0 +1,35 @@
+module Ration
+  module Backends
+    class Base
+      DEFAULT_MAX_PAYLOAD_BYTES = 6 * 1024
+
+      def publish(event)
+        raise NotImplementedError
+      end
+
+      def on_event(&block)
+        @on_event = block
+      end
+
+      def start
+        raise NotImplementedError
+      end
+
+      def stop
+        raise NotImplementedError
+      end
+
+      protected
+
+      def emit(event)
+        @on_event&.call(event)
+      end
+
+      def check_payload_size!(payload, limit)
+        return if payload.bytesize <= limit
+
+        raise PayloadTooLarge, "Payload is #{payload.bytesize} bytes, max is #{limit}"
+      end
+    end
+  end
+end

data/lib/ration/backends/memory.rb

@@ -0,0 +1,52 @@
+require 'json'
+require 'logger'
+
+module Ration
+  module Backends
+    class Memory < Base
+      def initialize(max_payload_bytes: DEFAULT_MAX_PAYLOAD_BYTES, sync: false, logger: nil)
+        super()
+        @max_payload_bytes = max_payload_bytes
+        @sync              = sync
+        @logger            = logger || Logger.new($stderr)
+        @queue             = nil
+        @thread            = nil
+      end
+
+      def publish(event)
+        check_payload_size!(event.to_json, @max_payload_bytes)
+
+        if @sync
+          emit(event)
+        else
+          @queue.push(event)
+        end
+      end
+
+      def start
+        return if @sync
+        return if @thread
+
+        @queue  = Queue.new
+        @thread = Thread.new {
+          while (event = @queue.pop)
+            begin
+              emit(event)
+            rescue => e
+              @logger.error("Ration::Backends::Memory listener error: #{e.class}: #{e.message}")
+            end
+          end
+        }
+      end
+
+      def stop
+        return if @sync
+
+        @queue&.close
+        @thread&.join
+        @queue  = nil
+        @thread = nil
+      end
+    end
+  end
+end

data/lib/ration/backends/postgres.rb

@@ -0,0 +1,118 @@
+require 'json'
+require 'logger'
+require 'pg'
+
+module Ration
+  module Backends
+    class Postgres < Base
+      DEFAULT_CHANNEL         = 'ration'
+      INITIAL_BACKOFF_SECONDS = 1
+      MAX_BACKOFF_SECONDS     = 30
+      DEFAULT_POLL_INTERVAL   = 1.0
+
+      def initialize(
+        url:,
+        channel:           DEFAULT_CHANNEL,
+        max_payload_bytes: DEFAULT_MAX_PAYLOAD_BYTES,
+        poll_interval:     DEFAULT_POLL_INTERVAL,
+        publish_with:      nil,
+        logger:            nil
+      )
+        super()
+        @url               = url
+        @channel           = channel
+        @max_payload_bytes = max_payload_bytes
+        @poll_interval     = poll_interval
+        @publish_with      = publish_with
+        @logger            = logger || Logger.new($stderr)
+        @thread            = nil
+        @stop              = false
+      end
+
+      def publish(event)
+        payload = event.to_json
+        check_payload_size!(payload, @max_payload_bytes)
+
+        if @publish_with
+          @publish_with.call(@channel, payload)
+        else
+          conn = PG.connect(@url)
+          begin
+            conn.exec_params('SELECT pg_notify($1, $2)', [@channel, payload])
+          ensure
+            conn.close
+          end
+        end
+      end
+
+      def start
+        return if @thread
+
+        @stop = false
+        initial_conn = connect_and_listen
+        @thread = Thread.new { run_loop(initial_conn) }
+      end
+
+      def stop
+        @stop = true
+        @thread&.join
+        @thread = nil
+      end
+
+      private
+
+      def connect_and_listen
+        conn = PG.connect(@url)
+        conn.exec("LISTEN #{conn.escape_identifier(@channel)}")
+        conn
+      end
+
+      def run_loop(conn)
+        current = conn
+
+        until @stop
+          begin
+            listen_loop(current)
+          rescue PG::Error => e
+            @logger.error("Ration::Backends::Postgres listener error: #{e.class}: #{e.message}")
+            current&.close
+            current = reconnect_with_backoff
+          end
+        end
+
+        current&.close
+      end
+
+      def reconnect_with_backoff
+        backoff = INITIAL_BACKOFF_SECONDS
+
+        until @stop
+          sleep(backoff)
+          break if @stop
+
+          begin
+            return connect_and_listen
+          rescue PG::Error => e
+            @logger.error("Ration::Backends::Postgres reconnect failed: #{e.class}: #{e.message}")
+            backoff = [backoff * 2, MAX_BACKOFF_SECONDS].min
+          end
+        end
+
+        nil
+      end
+
+      def listen_loop(conn)
+        until @stop
+          conn.wait_for_notify(@poll_interval) do |_channel, _pid, payload|
+            begin
+              event = JSON.parse(payload, symbolize_names: true)
+              emit(event)
+            rescue JSON::ParserError => e
+              @logger.error("Ration::Backends::Postgres received invalid JSON: #{e.message}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ration/backends/redis.rb

@@ -0,0 +1,130 @@
+require 'json'
+require 'logger'
+require 'redis-client'
+
+module Ration
+  module Backends
+    class Redis < Base
+      DEFAULT_CHANNEL         = 'ration'
+      INITIAL_BACKOFF_SECONDS = 1
+      MAX_BACKOFF_SECONDS     = 30
+      DEFAULT_POLL_INTERVAL   = 1.0
+
+      def initialize(
+        url:,
+        channel:           DEFAULT_CHANNEL,
+        max_payload_bytes: DEFAULT_MAX_PAYLOAD_BYTES,
+        poll_interval:     DEFAULT_POLL_INTERVAL,
+        publish_with:      nil,
+        logger:            nil
+      )
+        super()
+        @url               = url
+        @channel           = channel
+        @max_payload_bytes = max_payload_bytes
+        @poll_interval     = poll_interval
+        @publish_with      = publish_with
+        @logger            = logger || Logger.new($stderr)
+        @config            = RedisClient.config(url: url)
+        @thread            = nil
+        @stop              = false
+      end
+
+      def publish(event)
+        payload = event.to_json
+        check_payload_size!(payload, @max_payload_bytes)
+
+        if @publish_with
+          @publish_with.call(@channel, payload)
+        else
+          client = @config.new_client
+          begin
+            client.call('PUBLISH', @channel, payload)
+          ensure
+            client.close
+          end
+        end
+      end
+
+      def start
+        return if @thread
+
+        @stop = false
+        initial = subscribe_client
+        @thread = Thread.new { run_loop(initial) }
+      end
+
+      def stop
+        @stop = true
+        @thread&.join
+        @thread = nil
+      end
+
+      private
+
+      def subscribe_client
+        client = @config.new_client
+        pubsub = client.pubsub
+        pubsub.call('SUBSCRIBE', @channel)
+        pubsub
+      end
+
+      def run_loop(initial)
+        current = initial
+
+        until @stop
+          begin
+            listen_loop(current)
+          rescue RedisClient::Error => e
+            @logger.error("Ration::Backends::Redis listener error: #{e.class}: #{e.message}")
+            close_quietly(current)
+            current = reconnect_with_backoff
+          end
+        end
+
+        close_quietly(current)
+      end
+
+      def reconnect_with_backoff
+        backoff = INITIAL_BACKOFF_SECONDS
+
+        until @stop
+          sleep(backoff)
+          break if @stop
+
+          begin
+            return subscribe_client
+          rescue RedisClient::Error => e
+            @logger.error("Ration::Backends::Redis reconnect failed: #{e.class}: #{e.message}")
+            backoff = [backoff * 2, MAX_BACKOFF_SECONDS].min
+          end
+        end
+
+        nil
+      end
+
+      def listen_loop(pubsub)
+        until @stop
+          event = pubsub.next_event(@poll_interval)
+          next if event.nil?
+
+          type, _channel, payload = event
+          next unless type == 'message'
+
+          begin
+            parsed = JSON.parse(payload, symbolize_names: true)
+            emit(parsed)
+          rescue JSON::ParserError => e
+            @logger.error("Ration::Backends::Redis received invalid JSON: #{e.message}")
+          end
+        end
+      end
+
+      def close_quietly(pubsub)
+        pubsub&.close
+      rescue RedisClient::Error
+        # already broken; ignore
+      end
+    end
+  end
+end

data/lib/ration/configuration.rb

@@ -0,0 +1,12 @@
+require 'logger'
+
+module Ration
+  class Configuration
+    attr_accessor :backend, :logger
+
+    def initialize
+      @backend = nil
+      @logger  = Logger.new($stderr)
+    end
+  end
+end

data/lib/ration/errors.rb

@@ -0,0 +1,7 @@
+module Ration
+  class Error < StandardError; end
+
+  class PayloadTooLarge < Error; end
+
+  class NotConfigured < Error; end
+end

data/lib/ration/hub.rb

@@ -0,0 +1,86 @@
+require 'concurrent'
+
+module Ration
+  class Hub
+    def initialize(backend:, logger:)
+      @backend       = backend
+      @logger        = logger
+      @subscriptions = Concurrent::Map.new
+      @started       = false
+      @start_mutex   = Mutex.new
+    end
+
+    def publish(event)
+      ensure_started
+      @backend.publish(event)
+    end
+
+    def subscribe(max:, filter: nil, on_overflow: :close)
+      ensure_started
+
+      sub = Subscription.new(
+        max:         max,
+        filter:      filter,
+        on_overflow: on_overflow,
+        logger:      @logger
+      )
+      @subscriptions[sub.id] = sub
+
+      return sub unless block_given?
+
+      begin
+        yield sub
+      ensure
+        unsubscribe(sub)
+      end
+    end
+
+    def unsubscribe(sub)
+      @subscriptions.delete(sub.id)
+      sub.close
+    end
+
+    def stop
+      @start_mutex.synchronize do
+        @subscriptions.each_value(&:close)
+        @subscriptions.clear
+        @backend.stop if @started
+        @started = false
+      end
+    end
+
+    def subscription_count
+      @subscriptions.size
+    end
+
+    private
+
+    def ensure_started
+      return if @started
+
+      @start_mutex.synchronize do
+        return if @started
+
+        @backend.on_event {|event| deliver(event) }
+        @backend.start
+        @started = true
+      end
+    end
+
+    def deliver(event)
+      closed_ids = []
+
+      @subscriptions.each_pair do |id, sub|
+        begin
+          sub.deliver(event)
+        rescue => e
+          @logger.error("Ration delivery error for subscription #{id}: #{e.class}: #{e.message}")
+        end
+
+        closed_ids << id if sub.closed?
+      end
+
+      closed_ids.each {|id| @subscriptions.delete(id) }
+    end
+  end
+end

data/lib/ration/sse.rb

@@ -0,0 +1,94 @@
+require 'json'
+
+module Ration
+  module SSE
+    DEFAULT_ID_FROM = ->(event) { event[:id] }
+
+    class << self
+      def event(data:, event: nil, id: nil, retry_ms: nil)
+        raise ArgumentError, 'data is required' if data.nil?
+
+        lines = []
+
+        if event
+          ensure_field_value_safe!(event.to_s, 'event')
+          lines << "event: #{event}"
+        end
+
+        if id
+          ensure_field_value_safe!(id.to_s, 'id')
+          lines << "id: #{id}"
+        end
+
+        unless retry_ms.nil?
+          unless retry_ms.is_a?(Integer) && retry_ms >= 0
+            raise ArgumentError, 'retry_ms must be a non-negative Integer'
+          end
+
+          lines << "retry: #{retry_ms}"
+        end
+
+        payload = data.is_a?(String) ? data : data.to_json
+        payload.split("\n", -1).each do |line|
+          lines << "data: #{line}"
+        end
+
+        lines.join("\n") + "\n\n"
+      end
+
+      def comment(text = '')
+        ensure_no_newline!(text.to_s, 'comment')
+        ": #{text}\n\n"
+      end
+
+      def ping
+        ": ping\n\n"
+      end
+
+      def stream(
+        subscription,
+        output,
+        heartbeat: 15,
+        since:     nil,
+        id_from:   DEFAULT_ID_FROM
+      )
+        raise ArgumentError, 'block required' unless block_given?
+
+        last = since
+
+        subscription.each_event(timeout: heartbeat) do |event|
+          if event.nil?
+            output << ping if heartbeat
+            next
+          end
+
+          if last
+            id = id_from.call(event)
+            next if id.nil? || id <= last
+
+            last = id
+          end
+
+          framed = yield(event)
+          output << framed if framed
+        end
+
+        last
+      end
+
+      private
+
+      def ensure_field_value_safe!(value, field)
+        if value.match?(/[\r\n\0]/)
+          raise ArgumentError, "#{field} must not contain newlines or NULL characters"
+        end
+      end
+
+      def ensure_no_newline!(value, field)
+        if value.match?(/[\r\n]/)
+          raise ArgumentError, "#{field} must not contain newlines"
+        end
+      end
+    end
+  end
+end

data/lib/ration/subscription.rb

@@ -0,0 +1,87 @@
+require 'securerandom'
+
+module Ration
+  class Subscription
+    OVERFLOW_POLICIES = %i[close drop_oldest].freeze
+
+    attr_reader :id
+
+    def initialize(max:, filter: nil, on_overflow: :close, logger:)
+      unless OVERFLOW_POLICIES.include?(on_overflow)
+        raise ArgumentError, "Unknown on_overflow: #{on_overflow.inspect} (expected one of #{OVERFLOW_POLICIES.inspect})"
+      end
+
+      @id          = SecureRandom.uuid
+      @queue       = SizedQueue.new(max)
+      @filter      = filter
+      @on_overflow = on_overflow
+      @logger      = logger
+    end
+
+    def pop(timeout: nil)
+      @queue.pop(timeout: timeout)
+    end
+
+    def each_event(timeout: nil)
+      return enum_for(:each_event, timeout: timeout) unless block_given?
+
+      until closed?
+        event = pop(timeout: timeout)
+        break if closed?
+
+        yield event
+      end
+
+      self
+    end
+
+    def closed?
+      @queue.closed?
+    end
+
+    def close
+      @queue.close
+    end
+
+    def deliver(event)
+      return if closed?
+      return unless passes_filter?(event)
+
+      begin
+        @queue.push(event, true)
+      rescue ThreadError
+        handle_overflow(event)
+      rescue ClosedQueueError
+        # closed concurrently; nothing to do
+      end
+    end
+
+    private
+
+    def passes_filter?(event)
+      return true if @filter.nil?
+
+      @filter.call(event)
+    rescue => e
+      @logger.error("Ration filter raised, closing subscription #{@id}: #{e.class}: #{e.message}")
+      close
+      false
+    end
+
+    def handle_overflow(event)
+      case @on_overflow
+      when :close
+        close
+      when :drop_oldest
+        begin
+          @queue.pop(true)
+          @queue.push(event, true)
+        rescue ThreadError
+          # race: another thread popped first, or queue got closed; ignore
+        rescue ClosedQueueError
+          # closed concurrently; ignore
+        end
+      end
+    end
+  end
+end

data/lib/ration/version.rb

@@ -0,0 +1,3 @@
+module Ration
+  VERSION = '0.1.0'
+end

data/lib/ration.rb

@@ -0,0 +1,52 @@
+require_relative 'ration/version'
+require_relative 'ration/errors'
+require_relative 'ration/configuration'
+require_relative 'ration/subscription'
+require_relative 'ration/hub'
+require_relative 'ration/backends/base'
+require_relative 'ration/backends/memory'
+
+module Ration
+  class << self
+    def configure
+      yield config
+
+      if config.backend.nil?
+        raise NotConfigured, 'No backend configured. Set config.backend to a Ration::Backends::* instance.'
+      end
+
+      @hub&.stop
+      @hub = Hub.new(backend: config.backend, logger: config.logger)
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def publish(event)
+      hub.publish(event)
+    end
+
+    def subscribe(**kwargs, &block)
+      hub.subscribe(**kwargs, &block)
+    end
+
+    def unsubscribe(sub)
+      hub.unsubscribe(sub)
+    end
+
+    def reset!
+      @hub&.stop
+      @hub    = nil
+      @config = nil
+    end
+
+    private
+
+    def hub
+      raise NotConfigured, 'Ration is not configured. Call Ration.configure first.' unless @hub
+
+      @hub
+    end
+  end
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: ration
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Keita Urashima
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: 'A backend-agnostic event distribution layer for Rails SSE: a single
+  listener thread per process receives events from a pub/sub backend (Postgres LISTEN/NOTIFY,
+  Redis Pub/Sub, etc.) and fans them out to per-connection bounded queues.'
+email:
+- ursm@ursm.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/ration.rb
+- lib/ration/backends/base.rb
+- lib/ration/backends/memory.rb
+- lib/ration/backends/postgres.rb
+- lib/ration/backends/redis.rb
+- lib/ration/configuration.rb
+- lib/ration/errors.rb
+- lib/ration/hub.rb
+- lib/ration/sse.rb
+- lib/ration/subscription.rb
+- lib/ration/version.rb
+homepage: https://github.com/ursm/ration
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.10
+specification_version: 4
+summary: Per-process pub/sub fan-out for Rails SSE
+test_files: []