solid_cable 4.0.2 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a295176c880ba62299a280e557d841741ad471d57262ba535a8328787bb24a5
-  data.tar.gz: 18755bf5c482a9445ac9669ae2c341d5adc5654032388d8aa5bafcc21d7ab95b
+  metadata.gz: c8c6572058c2cfbbfe229f6bc376f1459611b40005e66396c82b419282535263
+  data.tar.gz: 6a480dc962405e9c97ab4efa82e47132e26bbfbc2a159f67f3c484de25655092
 SHA512:
-  metadata.gz: d8ed4f55e62d3ced9d59c6024de2809360d2d9bd16e26dcfe0ae0a209e84d034c30857fd30d8aa2ecbd5410755de880b26ac2139a19346eeff3a40f5fce26bff
-  data.tar.gz: 218429e22a991e5697d3431cc95510a68071eb1b9e03c5bd2107ae668eb26e13ac25e6d59a5d2b4ef898081eaff01fa3fc6f962dbaa699c866bb735b146d1660
+  metadata.gz: 35e2ecf675522441a1b4c8c088e63950aa2b7506a8cbdace50fae576e1de8902dbcb5e551587903e66648d1fc47ee8778480517631a37648ee9afcfbb96896a5
+  data.tar.gz: addcc08fd66aa27e4aa717b5ce1f1508efba528da6285489ba0676b2b044f37237ce065f8e2f6ab792d21bfc9989e5525550e6bd23da07cb4f0a8a6528af4fe0

data/README.md

@@ -88,11 +88,45 @@ The options are:
 - `trim_batch_size` - the batch size to use when deleting old records (default: `100`)
 - `reconnect_attempts` - Supports a number of connection attempts or an array of
   durations to wait between attempts. (Defaults to 1 retry attempt)
+- `encrypt` - whether to encrypt message payloads with Active Record Encryption.
+  (Defaults to false)
+
+### Enabling encryption
+
+Solid Cable can encrypt stored message payloads with Active Record Encryption. Add
+`encrypt: true` to the Solid Cable environment in `config/cable.yml`:
+
+```yaml
+production:
+  adapter: solid_cable
+  encrypt: true
+  connects_to:
+    database:
+      writing: cable
+```
+
+Your application must also be [configured to use Active Record Encryption](https://guides.rubyonrails.org/active_record_encryption.html#setup).
+Solid Cable uses the binary MessagePack serializer by default, so your application
+must include the `msgpack` gem.
+
+Since encryption context properties contain Ruby objects, they cannot be set in
+`config/cable.yml`. Set them in an initializer instead:
+
+```ruby
+# config/initializers/solid_cable.rb
+SolidCable.configuration.encryption_context_properties = {
+  encryptor: ActiveRecord::Encryption::Encryptor.new,
+  message_serializer: ActiveRecord::Encryption::MessageSerializer.new
+}
+```
+
+Active Record Encryption does not support encrypted binary columns on PostgreSQL
+with Rails 7. Solid Cable raises during boot for that unsupported combination.
 
 
 ## Trimming
 
-Messages are autotrimmed based upon the `message_retention` setting to determine how long messages are to be kept around. If no `message_retention` is given or parsing fails, it defaults to `1.day`. Messages are trimmed when a messsage is broadcast.
+Messages are autotrimmed based upon the `message_retention` setting to determine how long messages are to be kept around. If no `message_retention` is given or parsing fails, it defaults to `1.day`. For every message written, Solid Cable attempts to trim twice as many expired messages.
 
 Autotrimming can negatively impact performance slightly depending on your workload because it is potentially doing a delete on broadcast. If
 you would prefer, you can disable autotrimming by setting `autotrim: false` and you can manually enqueue the job later, `SolidCable::TrimJob.perform_later`, or run it on a recurring interval out of band.

data/app/jobs/solid_cable/trim_job.rb

@@ -2,26 +2,12 @@
 
 module SolidCable
   class TrimJob < ActiveJob::Base
-    def perform
-      return unless trim?
-
+    def perform(trim_batch_size: ::SolidCable.trim_batch_size)
       ::SolidCable::Message.transaction do
         ids = ::SolidCable::Message.trimmable.non_blocking_lock.
               limit(trim_batch_size).pluck(:id)
         ::SolidCable::Message.where(id: ids).delete_all
       end
     end
-
-    private
-      def trim_batch_size
-        ::SolidCable.trim_batch_size
-      end
-
-      def trim?
-        expires_per_write = (1 / trim_batch_size.to_f) * ::SolidCable.trim_chance
-
-        !::SolidCable.autotrim? ||
-          rand < (expires_per_write - expires_per_write.floor)
-      end
   end
 end

data/app/models/solid_cable/message/encryption.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module SolidCable
+  class Message
+    module Encryption
+      extend ActiveSupport::Concern
+
+      included do
+        if SolidCable.encrypt?
+          encrypts :payload, **SolidCable.encryption_context_properties, support_unencrypted_data: true
+        end
+      end
+    end
+  end
+end

data/app/models/solid_cable/message.rb

@@ -2,12 +2,14 @@
 
 module SolidCable
   class Message < SolidCable::Record
+    include Encryption
+
     scope :trimmable, lambda {
       where(created_at: ...::SolidCable.message_retention.ago)
     }
-    scope :broadcastable, lambda { |channels, last_id|
-      where(channel_hash: channel_hashes_for(channels)).
-        where(id: (last_id.to_i + 1)..).order(:id)
+    scope :broadcastable, lambda { |channel_hashes, last_id|
+      select(column_names.excluding("created_at", "updated_at")).
+        where(channel_hash: channel_hashes).where(id: (last_id.to_i + 1)..).order(:id)
     }
 
     class << self
@@ -16,8 +18,12 @@ module SolidCable
           channel_hash: channel_hash_for(channel) })
       end
 
-      def channel_hashes_for(channels)
-        channels.map { |channel| channel_hash_for(channel) }
+      def broadcast_batch(broadcasts)
+        created_at = Time.current
+        insert_all broadcasts.map { |message|
+          { created_at:, channel: message.channel,
+            payload: message.payload, channel_hash: channel_hash_for(message.channel) }
+        }
       end
 
       # Need to unpack this as a signed integer since Postgresql and SQLite

data/lib/action_cable/subscription_adapter/solid_cable.rb

@@ -20,23 +20,25 @@ module ActionCable
           end
 
         @listener = nil
+        @broadcaster = nil
       end
 
       def broadcast(channel, payload)
-        ::SolidCable::Message.broadcast(channel, payload)
-
-        ::SolidCable::TrimJob.perform_now if ::SolidCable.autotrim?
+        broadcaster.broadcast(channel, payload)
       end
 
-      def subscribe(channel, callback, success_callback = nil)
-        listener.add_subscriber(channel, callback, success_callback)
+      def subscribe(channel, subscriber, success_callback = nil)
+        listener.add_subscriber(channel, subscriber, success_callback)
       end
 
-      def unsubscribe(channel, callback)
-        listener.remove_subscriber(channel, callback)
+      def unsubscribe(channel, subscriber)
+        listener.remove_subscriber(channel, subscriber)
       end
 
-      delegate :shutdown, to: :listener
+      def shutdown
+        @broadcaster&.shutdown
+        @listener&.shutdown
+      end
 
       private
         def listener
@@ -45,6 +47,12 @@ module ActionCable
           end
         end
 
+        def broadcaster
+          @broadcaster || @mutex.synchronize do
+            @broadcaster ||= ::SolidCable::BatchedBroadcaster.new
+          end
+        end
+
         def pubsub_executor
           @pubsub_executor ||=
             if respond_to?(:executor, true)
@@ -122,12 +130,12 @@ module ActionCable
           end
 
           def add_channel(channel, on_success)
-            channels[channel] = last_message_id
+            channels[::SolidCable::Message.channel_hash_for(channel)] = last_message_id
             on_success.call if on_success
           end
 
           def remove_channel(channel)
-            channels.delete(channel)
+            channels.delete(::SolidCable::Message.channel_hash_for(channel))
           end
 
           def invoke_callback(*)
@@ -147,26 +155,27 @@ module ActionCable
             end
 
             def broadcast_messages
-              current_channels = channels.dup
-
               ::SolidCable::Message.
-                broadcastable(current_channels.keys, last_id).
-                each do |message|
+                broadcastable(channels.keys, last_id).each do |message|
                   should_broadcast_message = false
-                  channels.compute_if_present(message.channel) do |channel_last_id|
+                  channels.compute_if_present(message.channel_hash) do |channel_last_id|
                     break if channel_last_id >= message.id
 
                     should_broadcast_message = true
                     message.id
                   end
 
-                  broadcast(message.channel, message.payload) if should_broadcast_message
+                  broadcast(message) if should_broadcast_message
                   self.last_id = message.id
                 end
 
               self.reconnect_attempt = 0
             end
 
+            def broadcast(message)
+              super(message.channel, message.payload)
+            end
+
             def with_polling_volume
               if ::SolidCable.silence_polling? && ActiveRecord::Base.logger
                 ActiveRecord::Base.logger.silence { yield }

data/lib/generators/solid_cable/install/templates/config/cable.yml

@@ -15,3 +15,5 @@ production:
       writing: cable
   polling_interval: 0.1.seconds
   message_retention: 1.day
+  writer_batch_size: 4
+  writer_batch_delay: 0.001.seconds

data/lib/generators/solid_cable/install/templates/db/cable_schema.rb

@@ -4,7 +4,6 @@ ActiveRecord::Schema[7.1].define(version: 1) do
     t.binary "payload", limit: 536870912, null: false
     t.datetime "created_at", null: false
     t.integer "channel_hash", limit: 8, null: false
-    t.index ["channel"], name: "index_solid_cable_messages_on_channel"
     t.index ["channel_hash"], name: "index_solid_cable_messages_on_channel_hash"
     t.index ["created_at"], name: "index_solid_cable_messages_on_created_at"
   end

data/lib/generators/solid_cable/update/templates/db/migrate/remove_channel_index.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class RemoveChannelIndex < ActiveRecord::Migration[7.2]
+  def up
+    remove_index :solid_cable_messages, :channel, if_exists: true
+  end
+
+  def down
+    add_index :solid_cable_messages, :channel, if_not_exists: true
+  end
+end

data/lib/generators/solid_cable/update/update_generator.rb

@@ -11,5 +11,7 @@ class SolidCable::UpdateGenerator < Rails::Generators::Base
   def copy_files
     migration_template "db/migrate/create_compact_channel.rb",
                        "db/cable_migrate/create_compact_channel.rb"
+    migration_template "db/migrate/remove_channel_index.rb",
+                       "db/cable_migrate/remove_channel_index.rb"
   end
 end

data/lib/solid_cable/batched_broadcaster.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module SolidCable
+  class BatchedBroadcaster
+    include Trimming
+
+    Stopped = Class.new(StandardError)
+    Message = Struct.new(:channel, :payload, keyword_init: true)
+
+    def initialize(batch_size: SolidCable.writer_batch_size, batch_delay: SolidCable.writer_batch_delay)
+      @batch_size = batch_size
+      @batch_delay = batch_delay
+      @queue = Queue.new
+      @background = Concurrent::FixedThreadPool.new(1, max_queue: 100, fallback_policy: :discard)
+
+      @thread = Thread.new do
+        Thread.current.name = "solid_cable_writer"
+        Thread.current.abort_on_exception = true
+        listen_for_initial_messages
+      end
+    end
+
+    def broadcast(channel, payload)
+      message = Message.new(channel:, payload:)
+
+      queue.enq message
+    rescue ClosedQueueError
+      raise Stopped, "Solid Cable writer has stopped"
+    end
+
+    def shutdown
+      queue.close
+      thread.join
+      background.shutdown
+      background.wait_for_termination
+    end
+
+    private
+      attr_reader :batch_size, :batch_delay, :queue, :thread, :background
+
+      def listen_for_initial_messages
+        loop do
+          message = queue.pop
+
+          break if message.nil?
+
+          collect_batch(message)
+        end
+      end
+
+      def collect_batch(first_message)
+        batch = [ first_message ]
+        deadline = monotonic_time + batch_delay
+
+        drain_queue_into(batch)
+        wait_for_messages_until(batch, deadline)
+
+        flush(batch)
+      end
+
+      def drain_queue_into(batch)
+        while batch.size < batch_size && (message = queue.pop(timeout: 0))
+          batch << message
+        end
+      end
+
+      def wait_for_messages_until(batch, deadline)
+        while batch.size < batch_size && (remaining = deadline - monotonic_time).positive?
+          message = queue.pop(timeout: remaining)
+          break if message.nil?
+
+          batch << message
+        end
+      end
+
+      def flush(batch)
+        Rails.application.executor.wrap do
+          SolidCable::Message.broadcast_batch(batch)
+          track_writes(batch.size) if SolidCable.autotrim?
+        end
+      rescue StandardError => error
+        Rails.error.report(error)
+      end
+
+      def async(&block)
+        background << -> do
+          Rails.application.executor.wrap(&block)
+        rescue Exception => error # rubocop:disable Lint/RescueException
+          Rails.error.report(error)
+        end
+      end
+
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+  end
+end

data/lib/solid_cable/configuration.rb

@@ -0,0 +1,104 @@
+module SolidCable
+  class Configuration
+    def initialize(**options)
+      @options = ActiveSupport::InheritableOptions.new(options)
+    end
+
+    attr_writer :connects_to, :silence_polling, :polling_interval,
+      :message_retention, :autotrim, :trim_batch_size, :use_skip_locked,
+      :reconnect_attempts, :writer_batch_size, :writer_batch_delay,
+      :encrypt, :encryption_context_properties
+
+    def connects_to
+      @connects_to ||= options.connects_to.to_h.deep_transform_values(&:to_sym)
+    end
+
+    def silence_polling?
+      return @silence_polling if defined?(@silence_polling)
+
+      @silence_polling = options.silence_polling != false
+    end
+
+    def polling_interval
+      @polling_interval ||=
+        parse_duration(options.polling_interval, default: 0.1.seconds)
+    end
+
+    def message_retention
+      @message_retention ||= parse_duration(options.message_retention, default: 1.day)
+    end
+
+    def autotrim?
+      return @autotrim if defined?(@autotrim)
+
+      @autotrim = options.autotrim != false
+    end
+
+    def trim_batch_size
+      @trim_batch_size ||=
+        if (size = options.trim_batch_size.to_i) < 2
+          100
+        else
+          size
+        end
+    end
+
+    def use_skip_locked
+      return @use_skip_locked if defined?(@use_skip_locked)
+
+      @use_skip_locked = options.use_skip_locked != false
+    end
+
+    def reconnect_attempts
+      @reconnect_attempts ||= begin
+        attempts = options[:reconnect_attempts] || 1
+        attempts = Array.new(attempts, 0) if attempts.is_a?(Integer)
+        attempts
+      end
+    end
+
+    def writer_batch_size
+      @writer_batch_size ||= [ (options.writer_batch_size || 4).to_i, 1 ].max
+    end
+
+    def writer_batch_delay
+      @writer_batch_delay ||=
+        [ parse_duration(options.writer_batch_delay, default: 0.001.seconds), 0 ].max
+    end
+
+    def encrypt?
+      return @encrypt if defined?(@encrypt)
+
+      @encrypt = options.encrypt.present?
+    end
+
+    def encryption_context_properties
+      return @encryption_context_properties if defined?(@encryption_context_properties)
+
+      @encryption_context_properties = options.encryption_context_properties&.deep_symbolize_keys
+      @encryption_context_properties ||= default_encryption_context_properties if encrypt?
+    end
+
+    private
+      attr_reader :options
+
+      def default_encryption_context_properties
+        require "active_record/encryption/message_pack_message_serializer"
+
+        {
+          encryptor: ActiveRecord::Encryption::Encryptor.new(compress: true),
+          # Binary column only serializer that is 40% more efficient than the default MessageSerializer
+          message_serializer: ActiveRecord::Encryption::MessagePackMessageSerializer.new
+        }
+      end
+
+      def parse_duration(duration, default:)
+        if duration.present?
+          *amount, units = duration.to_s.split(".")
+          amount.join(".").to_f.public_send(units)
+        else
+          default
+        end
+      end
+  end
+end

data/lib/solid_cable/engine.rb

@@ -3,5 +3,13 @@
 module SolidCable
   class Engine < ::Rails::Engine
     isolate_namespace SolidCable
+
+    config.after_initialize do
+      if SolidCable.encrypt? && Record.lease_connection.adapter_name == "PostgreSQL" && Rails::VERSION::MAJOR == 7
+        raise \
+          "Cannot enable encryption for Solid Cable: in Rails 7, Active Record Encryption does not support " \
+          "encrypting binary columns on PostgreSQL"
+      end
+    end
   end
 end

data/lib/solid_cable/trimming.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module SolidCable
+  module Trimming
+    # For every write that we do, we attempt to delete TRIM_MULTIPLIER times as
+    # many records. This ensures there is downward pressure on the message count
+    # while there is old data to delete.
+    TRIM_MULTIPLIER = 2
+
+    private
+      def track_writes(count)
+        trim_batches(count).times { async { TrimJob.perform_now } }
+      end
+
+      def trim_batches(count)
+        trims_per_write = (1 / SolidCable.trim_batch_size.to_f) * TRIM_MULTIPLIER
+        batches = (count * trims_per_write).floor
+        overflow_batch_chance = count * trims_per_write - batches
+        batches += 1 if rand < overflow_batch_chance
+        batches
+      end
+  end
+end

data/lib/solid_cable/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SolidCable
-  VERSION = "4.0.2"
+  VERSION = "4.1.0"
 end

data/lib/solid_cable.rb

@@ -2,70 +2,29 @@
 
 require "solid_cable/version"
 require "solid_cable/engine"
+require "solid_cable/configuration"
+require "solid_cable/trimming"
+require "solid_cable/batched_broadcaster"
 require "action_cable/subscription_adapter/solid_cable"
 
 module SolidCable
   class << self
-    def connects_to
-      cable_config.connects_to.to_h.deep_transform_values(&:to_sym)
-    end
-
-    def silence_polling?
-      cable_config.silence_polling != false
-    end
-
-    def polling_interval
-      parse_duration(cable_config.polling_interval, default: 0.1.seconds)
-    end
-
-    def message_retention
-      parse_duration(cable_config.message_retention, default: 1.day)
-    end
+    delegate :connects_to, :silence_polling?, :polling_interval,
+      :message_retention, :autotrim?, :trim_batch_size, :use_skip_locked,
+      :reconnect_attempts, :writer_batch_size, :writer_batch_delay,
+      :encrypt?, :encryption_context_properties,
+      to: :configuration
 
-    def autotrim?
-      cable_config.autotrim != false
+    def configuration
+      @configuration ||= Configuration.new(**Rails.application.config_for("cable"))
     end
 
-    def trim_batch_size
-      if (size = cable_config.trim_batch_size.to_i) < 2
-        100
-      else
-        size
-      end
+    def configure(**options)
+      @configuration = Configuration.new(**options)
     end
 
-    def use_skip_locked
-      cable_config.use_skip_locked != false
+    def reset_configuration!
+      @configuration = nil
     end
-
-    # For every write that we do, we attempt to delete trim_chance times as
-    # many records. This ensures there is downward pressure on the cache size
-    # while there is valid data to delete. Read this as 'every time the trim job
-    # runs theres a trim_multiplier chance this trims'. Adjust number to make it
-    # more or less likely to trim. Only works like this if trim_batch_size is
-    # 100
-    def trim_chance
-      2
-    end
-
-    def reconnect_attempts
-      attempts = cable_config.fetch(:reconnect_attempts, 1)
-      attempts = Array.new(attempts, 0) if attempts.is_a?(Integer)
-      attempts
-    end
-
-    private
-      def cable_config
-        Rails.application.config_for("cable")
-      end
-
-      def parse_duration(duration, default:)
-        if duration.present?
-          *amount, units = duration.to_s.split(".")
-          amount.join(".").to_f.public_send(units)
-        else
-          default
-        end
-      end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid_cable
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.1.0
 platform: ruby
 authors:
 - Nick Pezza
@@ -65,6 +65,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.2'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -91,6 +105,7 @@ files:
 - Rakefile
 - app/jobs/solid_cable/trim_job.rb
 - app/models/solid_cable/message.rb
+- app/models/solid_cable/message/encryption.rb
 - app/models/solid_cable/record.rb
 - lib/action_cable/subscription_adapter/solid_cable.rb
 - lib/generators/solid_cable/install/USAGE
@@ -99,9 +114,13 @@ files:
 - lib/generators/solid_cable/install/templates/db/cable_schema.rb
 - lib/generators/solid_cable/update/USAGE
 - lib/generators/solid_cable/update/templates/db/migrate/create_compact_channel.rb
+- lib/generators/solid_cable/update/templates/db/migrate/remove_channel_index.rb
 - lib/generators/solid_cable/update/update_generator.rb
 - lib/solid_cable.rb
+- lib/solid_cable/batched_broadcaster.rb
+- lib/solid_cable/configuration.rb
 - lib/solid_cable/engine.rb
+- lib/solid_cable/trimming.rb
 - lib/solid_cable/version.rb
 - lib/tasks/solid_cable_tasks.rake
 homepage: https://github.com/rails/solid_cable
@@ -125,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.15
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Database-backed Action Cable backend.
 test_files: []