cable_room 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bc7b6f7f0f2c70e5db315cd4dc11ff01e7b0d8dcd56ba5d54159d2c0f3b6d56e
+  data.tar.gz: b8a9866e23a63d93cd52c88ce9008d194f2ed3cea2060b0b533c5236235fe244
+SHA512:
+  metadata.gz: c13aee8af8dba031b0e7333539bde585f8e898a8afbf87657a0fc0128e21c32580149b866107c976c4742df690f8c838d0e094b63e96d140a1d6fb6731a8d652
+  data.tar.gz: 2ca7afb01a2bf5b2c0096045f6f486ecf8a744b63017497fcf9dd835d770d1fd99b37d63a8ff8c442e1aacd7ed5e0167725f08077c93a6de3fde13a94805c4bd

data/README.md

@@ -0,0 +1 @@
+# RediConn

data/cable_room.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+begin
+  require "cable_room/version"
+  version = CableRoom::VERSION
+rescue LoadError
+  version = "0.0.0.docker"
+end
+
+Gem::Specification.new do |spec|
+  spec.name          = "cable_room"
+  spec.version       = version
+  spec.authors       = ["Ethan Knapp"]
+  spec.email         = ["eknapp@instructure.com"]
+
+  spec.summary       = "Build live Rooms on top of ActionCable"
+  spec.homepage      = "https://instructure.com"
+
+  spec.files = Dir["{app,config,db,lib}/**/*", "README.md", "*.gemspec"]
+  spec.test_files = Dir["spec/**/*"]
+  spec.require_paths = ['lib']
+
+  spec.add_dependency "rails", ">= 7.2", "< 9.0"
+  spec.add_dependency "rufus-scheduler", "~> 3.6"
+  spec.add_dependency "redlock", "~> 2.0"
+  spec.add_dependency "rediconn", "~> 0.1.0"
+
+  spec.add_development_dependency "redis"
+  spec.add_development_dependency 'rspec', '~> 3'
+end

data/lib/cable_room/channel_base.rb

@@ -0,0 +1,194 @@
+module CableRoom
+  class ChannelBase < ActionCable::Channel::Base
+    attr_reader :room, :tenant
+    attr_reader :server, :logger
+    delegate :event_loop, :worker_pool, :pubsub, to: :server
+
+    def initialize(lock_info, room_class, key, config)
+      # We don't really have a "connection" in the ActionCable sense, so
+      #   stuff in something that half looks like one
+      super(DummyConnection.new(self), "Room[]", {})
+
+      # Used mainly for logs and being able to follow a specific Room instance
+      @uuid = SecureRandom.hex(6)
+
+      @mutex = Monitor.new
+      @lock_info = lock_info
+      @current_state = :initializing
+
+      @tenant = Apartment::Tenant.current if defined?(Apartment)
+
+      @server = ChannelTracker.instance
+      @logger = ActionCable::Connection::TaggedLoggerProxy.new(
+        @server.logger,
+        tags: ["#{room_class.name} #{@uuid}"]
+      )
+
+      logger.info "Initializing new #{room_class.name}"
+      logger.info "  UUID: #{@uuid}"
+      logger.info "  Key: #{room_class.room_port_key(key)}"
+
+      @watchdog_interval = config[:watchdog_interval]
+      @lock_duration = config[:lock_duration]
+
+      @processing_work = false
+      @work_queue = []
+
+      @room = room_class.new(self, key)
+      @server.track_room_channel self
+      ping_watchdog
+    end
+
+    def self.channel_name
+      module_parent.name
+    end
+
+    def stream_from(...)
+      raise ArgumentError, "Block required" unless block_given?
+      super
+    end
+
+    def state
+      @current_state
+    end
+
+    after_subscribe do
+      begin
+        @current_state = :starting
+        @room.send(:_startup)
+        @current_state = :started
+      rescue => e
+        terminate!
+        raise e
+      end
+    end
+
+    after_unsubscribe do
+      @mutex.synchronize do
+        @current_state = :shutting_down
+        stop_all_streams
+        @room.send(:_shutdown)
+        terminate!
+      end
+    end
+
+    def ping_watchdog
+      return if state == :dead
+
+      logger.debug "Ping watchdog"
+      @last_watchdog_ping_at = Time.current
+    end
+
+    def check_room_watchdog
+      return if state == :dead || state == :shutting_down
+
+      relock = CableRoom.lock_manager.lock(@lock_info[:resource], @lock_duration.in_milliseconds, extend: @lock_info)
+      unless relock
+        logger.warn "Lost lock, shutting down"
+        unsubscribe_from_channel
+        return
+      end
+
+      unless @last_watchdog_ping_at && @last_watchdog_ping_at > @watchdog_interval.ago
+        logger.warn "Watchdog timeout for room #{@room.class.name}[#{@room.key}], shutting down"
+        initiate_shutdown("Watchdog timeout")
+        return
+      end
+    end
+
+    def initiate_shutdown(reason)
+      @mutex.synchronize do
+        return if @current_state == :dead || @current_state == :shutting_down
+
+        # Stop streams immediately to prevent further messages from being added
+        stop_all_streams
+
+        # Append the final unsubscribe to the work queue so we can process remaining messages first
+        post_work(async: false) do
+          unsubscribe_from_channel
+        end
+
+        @current_state = :shutting_down
+      end
+    end
+
+    def terminate!
+      @mutex.synchronize do
+        stop_all_streams
+        @current_state = :dead
+        CableRoom.lock_manager.unlock(@lock_info) if @lock_info
+        server.untrack_room_channel self
+      end
+    end
+
+    def post_work(async: false, &blk)
+      if async
+        # Async stuff is mostly untracked - we just post it to the worker pool and forget about it
+        worker_pool.executor.post(&blk)
+      else
+        @mutex.synchronize do
+          raise "Attempt to post work to dead or shutting down room" if @current_state == :dead || @current_state == :shutting_down
+          @work_queue << blk
+        end
+        schedule_work
+      end
+    end
+
+    def beat
+      post_work(async: true) do
+        check_room_watchdog
+      end
+    end
+
+    def transmit(*args)
+      logger.info("Channel.transmit called, ignoring: #{args.inspect}")
+    end
+
+    protected
+
+    def start_periodic_timer(callback, every:)
+      raise "Attempt to start periodic timer on a dead room" if state == :dead || state == :shutting_down
+
+      connection.server.scheduler.schedule_every(every) do
+        post_work(async: false) do
+          instance_exec(&callback)
+        end
+      end
+    end
+
+    def schedule_work
+      @mutex.synchronize do
+        return if @processing_work
+
+        work = @work_queue.shift
+        return unless work
+
+        @processing_work = true
+
+        worker_pool.executor.post do
+          begin
+            work.call
+          ensure
+            @mutex.synchronize do
+              @processing_work = false
+            end
+            schedule_work
+          end
+        end
+      end
+    end
+  end
+
+  class DummyConnection
+    attr_reader :channel
+    delegate :server, :logger, :tenant, :transmit, :post_work, to: :channel
+    delegate :event_loop, :pubsub, :worker_pool, to: :server
+
+    attr_reader :identifiers
+
+    def initialize(channel)
+      @channel = channel
+      @identifiers = []
+    end
+  end
+end

data/lib/cable_room/channel_tracker.rb

@@ -0,0 +1,118 @@
+module CableRoom
+  class ChannelTracker
+    def self.instance
+      @instance ||= new
+    end
+
+    BEAT_INTERVAL = 5.seconds
+
+    delegate :logger, :pubsub, :event_loop, :config, to: :cable_server
+
+    attr_reader :room_channels, :scheduler
+
+    def initialize
+      @room_channels = Set.new
+      @monitor = Monitor.new
+
+      @scheduler = Rufus::Scheduler.new
+
+      at_exit do
+        logger.info "Shutting down CableRoom"
+        shutdown!
+      end
+
+      scheduler.every(BEAT_INTERVAL) do
+        each_room_channel do |chan|
+          worker_pool.async_invoke(chan, :beat)
+        end
+      end
+    end
+
+    def worker_pool
+      # TODO Pin Rooms to a specific thread so that they never have to worry about thread safety?
+      @worker_pool || @monitor.synchronize { @worker_pool ||= ThreadPool.new(max_size: config.worker_pool_size) }
+    end
+
+    def track_room_channel(chan)
+      raise "Cannot add Room after shutdown" if @shutdown
+      @room_channels << chan
+    end
+
+    def untrack_room_channel(chan)
+      @room_channels.delete(chan)
+    end
+
+    def shutdown?
+      @shutdown
+    end
+
+    def shutdown!
+      @monitor.synchronize do
+        @shutdown = true
+
+        scheduler.shutdown
+
+        shutdown_rooms!
+
+        worker_pool.executor.shutdown
+        worker_pool.executor.wait_for_termination(5)
+        worker_pool.executor.kill
+      end
+    end
+
+    def shutdown_rooms!
+      @monitor.synchronize do
+        each_room_channel do |chan|
+          chan.initiate_shutdown("Server shutting down")
+        end
+      end
+
+      count = 0
+      loop do
+        break if @room_channels.empty? || count >= 15
+        sleep 1
+        count += 1
+      end
+    end
+
+    protected
+
+    def each_room_channel(&blk)
+      @room_channels.dup.each(&blk)
+    end
+
+    private
+
+    def cable_server
+      ActionCable.server
+    end
+
+    class ThreadPool < ActionCable::Server::Worker
+      set_callback :work, :around do |_, blk|
+        pconn = ActionCable::Server::Worker.connection
+        ActionCable::Server::Worker.connection = connection
+        blk.call
+      ensure
+        ActionCable::Server::Worker.connection = pconn
+      end
+
+      def async_invoke(receiver, method, *args, connection: receiver, async: nil, &block)
+        # Instead of posting directly to the global pool, post to a dedicated queue for the room/"connection".
+        #   This makes each rooms so that they can be processed by at-most-one thread at a time, while still
+        #   allowing multiple rooms to be processed by the same thread-pool.
+
+        # TODO Implement more of a round-robin approach so that no room can starve out others?
+
+        # "connection" here really references the Channel, since "Connections" in this context don't really exist
+
+        connection.post_work(async:) do
+          invoke(receiver, method, *args, connection: connection, &block)
+        end
+      end
+    end
+
+    Rufus::Scheduler::Job.class_eval do
+      alias_method :shutdown, :unschedule
+    end
+  end
+end

data/lib/cable_room/ports.rb

@@ -0,0 +1,62 @@
+module CableRoom
+  module Ports
+    extend ActiveSupport::Concern
+
+    def ports
+      @ports_proxy ||= PortsProxy.new(self)
+    end
+
+    def stream_port(port, auto_close: true, &blk)
+      @cable_channel.stream_from(room_port_key(port), coder: ActiveSupport::JSON, &blk)
+      _streamed_ports << port if auto_close
+    end
+
+    def close_streamed_ports!
+      _streamed_ports.each do |port|
+        @cable_channel.stop_stream_from(room_port_key(port))
+      end
+      _streamed_ports.clear
+    end
+
+    def port_transmit(port, data)
+      ActionCable.server.broadcast(room_port_key(port), data)
+    end
+
+    protected
+
+    def _room_channel_class
+      room_class.channel_class
+    end
+
+    def _streamed_ports
+      @_streamed_ports ||= Set.new
+    end
+
+    class PortsProxy
+      def initialize(ports_concern)
+        @ports_concern = ports_concern
+      end
+
+      def [](port)
+        PortProxy.new(@ports_concern, port)
+      end
+    end
+
+    class PortProxy
+      def initialize(ports_concern, port)
+        @ports_concern = ports_concern
+        @port = port
+      end
+
+      def transmit(data)
+        @ports_concern.port_transmit(@port, data)
+      end
+
+      alias_method :<<, :transmit
+
+      def stream(&blk)
+        @ports_concern.stream_port(@port, &blk)
+      end
+    end
+  end
+end

data/lib/cable_room/railtie.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "rails"
+require "active_model/railtie"
+
+module CableRoom
+  class Railtie < Rails::Railtie # :nodoc:
+    rake_tasks do
+    end
+
+    console do |app|
+    end
+
+    runner do
+    end
+
+    initializer "cable_room.shutdown" do
+      Rails.application.reloader.before_class_unload do
+        CableRoom::ChannelTracker.instance.shutdown_rooms!
+      end
+    end
+
+    initializer "cable_room.monkey_patch_redlock" do
+      require 'redlock'
+
+      module ::Redlock
+        class Client
+          private
+          class RedisInstance
+            private
+            def recover_from_script_flush
+              retry_on_noscript = true
+              begin
+                yield
+              rescue RedisClient::CommandError, Redis::CommandError => e # <= ADDED Redis::CommandError
+                # When somebody has flushed the Redis instance's script cache, we might
+                # want to reload our scripts. Only attempt this once, though, to avoid
+                # going into an infinite loop.
+                if retry_on_noscript && e.message.include?('NOSCRIPT')
+                  load_scripts
+                  retry_on_noscript = false
+                  retry
+                else
+                  raise
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cable_room/room/base.rb

@@ -0,0 +1,92 @@
+module CableRoom
+  module Room
+    class Base
+      ROOM_OUT_CHANNEL = :from_room # Many-to-one channel for messages to the room
+      ROOM_IN_CHANNEL = :to_room # One-to-many channel for messages from the room
+
+      LOCK_DURATION = 15.seconds
+      WATCH_DOG_INTERVAL = 15.seconds
+
+      class << self
+        def ensure(key = nil)
+          lock_key = room_port_key(key)
+          lock_info = CableRoom.lock_manager.lock(lock_key, self::LOCK_DURATION.in_milliseconds)
+          return false unless lock_info
+
+          vchannel = channel_class.new(
+            lock_info,
+            self,
+            key,
+            {
+              watchdog_interval: self::WATCH_DOG_INTERVAL,
+              lock_duration: self::LOCK_DURATION,
+            }
+          )
+          vchannel.subscribe_to_channel
+
+          true
+        end
+
+        def room_port_key(room_key, port = nil)
+          full_key = [room_key]
+          full_key << port if port
+          self::Channel.broadcasting_for(full_key)
+        end
+
+        def send_message(room_key, data, port: ROOM_IN_CHANNEL)
+          ActionCable.server.broadcast(room_port_key(room_key, port), data)
+        end
+
+        protected
+
+        def periodically(method, every:, &blk)
+          blk ||= method
+          blk = -> { send(method) } unless blk.is_a?(Proc)
+          # TODO Looking at the docs, this creates a thread for each timer - that seems expensive
+          channel_class.periodically(every:) do
+            room.instance_exec(&blk)
+          end
+        end
+      end
+
+      include Ports
+      include Callbacks
+      include Threading
+      include ChannelAdapter
+      include InputHandling
+      include Lifecycle
+      include Reaping
+
+      include MemberManagement
+      include UserManagement
+
+      attr_reader :key
+
+      delegate :logger, :params, to: :@cable_channel
+
+      def initialize(vchannel, key = nil)
+        super()
+        @key = key
+        @cable_channel = vchannel
+      end
+
+      def <<(data)
+        port_transmit(ROOM_OUT_CHANNEL, data)
+      end
+
+      protected
+
+      def room_class
+        self.class
+      end
+
+      def room_port_key(sub_channel = nil)
+        self.class.room_port_key(key, sub_channel)
+      end
+
+      def start_periodic_timer(callback, every:)
+        @cable_channel.send(:start_periodic_timer, -> { room.instance_exec(&callback) }, every:)
+      end
+    end
+  end
+end

data/lib/cable_room/room/callbacks.rb

@@ -0,0 +1,33 @@
+module CableRoom
+  module Room
+    module Callbacks
+      extend  ActiveSupport::Concern
+      include ActiveSupport::Callbacks
+
+      included do
+        define_callbacks :startup
+        define_callbacks :shutdown
+      end
+
+      class_methods do
+        def before_startup(*methods, &block)
+          set_callback(:startup, :before, *methods, &block)
+        end
+
+        def after_startup(*methods, &block)
+          set_callback(:startup, :after, *methods, &block)
+        end
+        alias_method :on_startup, :after_startup
+
+        def before_shutdown(*methods, &block)
+          set_callback(:shutdown, :before, *methods, &block)
+        end
+
+        def after_shutdown(*methods, &block)
+          set_callback(:shutdown, :after, *methods, &block)
+        end
+        alias_method :on_shutdown, :after_shutdown
+      end
+    end
+  end
+end

data/lib/cable_room/room/channel_adapter.rb

@@ -0,0 +1,20 @@
+module CableRoom
+  module Room
+    module ChannelAdapter
+      extend ActiveSupport::Concern
+
+      Channel = CableRoom::ChannelBase
+
+      class_methods do
+        def inherited(subclass)
+          subclass.const_set(:Channel, Class.new(channel_class))
+          super
+        end
+
+        def channel_class
+          self::Channel
+        end
+      end
+    end
+  end
+end

data/lib/cable_room/room/input_handling.rb

@@ -0,0 +1,31 @@
+module CableRoom
+  module Room
+    module InputHandling
+      extend ActiveSupport::Concern
+
+      included do
+        before_startup do
+          ports[self.class::ROOM_IN_CHANNEL].stream do |message|
+            logger.debug "Received message: #{message.inspect}"
+            if message == "KILL"
+              shutdown!
+            else
+              handle_received_message(message)
+            end
+          end
+        end
+      end
+
+      protected
+
+      def handle_received_message(message)
+        sym = :"on_#{message['type'].underscore.to_s.downcase}"
+        if respond_to?(sym, true)
+          send(sym, message)
+        else
+          logger.warn "Unknown room message type: #{message['type'].inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/cable_room/room/lifecycle.rb

@@ -0,0 +1,53 @@
+module CableRoom
+  module Room
+    module Lifecycle
+      extend ActiveSupport::Concern
+
+      included do
+        after_startup do
+          logger.info "Started"
+          self << { type: 'room_opened' }
+        end
+
+        before_shutdown do
+          self << { type: 'room_closed' }
+          logger.info "Shutting down"
+        end
+
+        after_shutdown do
+          logger.info "Shutdown complete"
+        end
+      end
+
+      protected
+
+      def lifecycle_state
+        @cable_channel.state
+      end
+
+      # Requests that the Room shut down gracefully, processing any pending messages
+      def shutdown!(reason = "Room requested shutdown")
+        @cable_channel.initiate_shutdown(reason)
+      end
+
+      # Stops the room synchronously, ignoring remaining messages
+      def stop!
+        @cable_channel.unsubscribe_from_channel
+      end
+
+      private
+
+      def _startup
+        run_callbacks :startup do
+          startup if respond_to?(:startup)
+        end
+      end
+
+      def _shutdown
+        run_callbacks :shutdown do
+          shutdown if respond_to?(:shutdown)
+        end
+      end
+    end
+  end
+end