redstream 0.0.1

data/lib/redstream/consumer.rb

@@ -0,0 +1,115 @@
+
+require "thread"
+
+module Redstream
+  # The Redstream::Consumer class to read messages from a specified redis
+  # stream in batches.
+  #
+  # @example
+  #   Redstream::Consumer.new(name: "user_indexer", stream_name: "users").run do |messages|
+  #     # ...
+  #   end
+
+  class Consumer
+    # Initializes a new consumer instance. Please note that you can have
+    # multiple consumers per stream, by specifying different names.
+    #
+    # @param name [String] The consumer name. The name is used for locking
+    # @param stream_name [String] The name of the redis stream. Please note
+    #   that redstream adds a prefix to the redis keys. However, the
+    #   stream_name param must be specified without any prefixes here.
+    #   When using Redstream::Model, the stream name is the downcased,
+    #   pluralized and underscored version of the model name. I.e., the
+    #   stream name for a 'User' model will be 'users'
+    # @param batch_size [Fixnum] The desired batch size, that is the number
+    #   of messages yielded at max. More concretely, the number of messages
+    #   yielded may be lower the batch_size, but not higher
+    # @param logger [Logger] The logger used for error logging
+
+    def initialize(name:, stream_name:, batch_size: 1_000, logger: Logger.new("/dev/null"))
+      @name = name
+      @stream_name = stream_name
+      @batch_size = batch_size
+      @logger = logger
+      @redis = Redstream.connection_pool.with(&:dup)
+      @lock = Lock.new(name: "consumer:#{@stream_name}:#{@name}")
+    end
+
+    # Returns its maximum committed id, i.e. the consumer's offset.
+    #
+    # @return [String, nil] The committed id, or nil
+
+    def max_committed_id
+      @redis.get Redstream.offset_key_name(stream_name: @stream_name, consumer_name: @name)
+    end
+
+    # Loops and thus blocks forever while reading messages from the specified
+    # stream and yielding them in batches.
+    #
+    # @example
+    #   consumer.run do |messages|
+    #     # ...
+    #   end
+
+    def run(&block)
+      loop { run_once(&block) }
+    end
+
+    # Reads a single batch from the specified stream and yields it. You usually
+    # want to use the #run method instead, which loops/blocks forever.
+    #
+    # @example
+    #   consumer.run_once do |messages|
+    #     # ...
+    #   end
+
+    def run_once(&block)
+      got_lock = @lock.acquire do
+        offset = @redis.get(Redstream.offset_key_name(stream_name: @stream_name, consumer_name: @name))
+        offset ||= "0-0"
+
+        stream_key_name = Redstream.stream_key_name(@stream_name)
+
+        response = begin
+          @redis.xread(stream_key_name, offset, count: @batch_size, block: 5_000)
+        rescue Redis::TimeoutError
+          nil
+        end
+
+        return if response.nil? || response[stream_key_name].nil? || response[stream_key_name].empty?
+
+        messages = response[stream_key_name].map do |raw_message|
+          Message.new(raw_message)
+        end
+
+        block.call(messages)
+
+        offset = response[stream_key_name].last[0]
+
+        return unless offset
+
+        commit offset
+      end
+
+      sleep(5) unless got_lock
+    rescue => e
+      @logger.error e
+
+      sleep 5
+
+      retry
+    end
+
+    # @api private
+    #
+    # Commits the specified offset/ID as the maximum ID already read, such that
+    # subsequent read calls will use this offset/ID as a starting point.
+    #
+    # @param offset [String] The offset/ID to commit
+
+    def commit(offset)
+      @redis.set Redstream.offset_key_name(stream_name: @stream_name, consumer_name: @name), offset
+    end
+  end
+end
+

data/lib/redstream/delayer.rb

@@ -0,0 +1,100 @@
+
+module Redstream
+  # The Redstream::Delayer class is responsible for reading messages from
+  # special delay streams which are used to fix inconsistencies resulting from
+  # network or other issues in between after_save and after_commit callbacks.
+  # To be able to fix such issues, delay messages will be added to a delay
+  # stream within an after_save callback. The delay messages aren't fetched
+  # immediately, but e.g. 5 minutes later, such that we can be sure that the
+  # database transaction is committed or has been rolled back, but is no longer
+  # running.
+  #
+  # @example
+  #   Redstream::Delayer.new(stream_name: "users", delay: 5.minutes, logger: Logger.new(STDOUT)).run
+
+  class Delayer
+    # Initializes the delayer for the specified stream name and delay.
+    #
+    # @param stream_name [String] The stream name. Please note, that redstream
+    #   adds a prefix to the redis keys. However, the stream_name param must
+    #   be specified without any prefixes here. When using Redstream::Model,
+    #   the stream name is the downcased, pluralized and underscored version
+    #   of the model name. I.e., the stream name for a 'User' model will be
+    #   'users'
+    # @param delay [Fixnum, Float, ActiveSupport::Duration] The delay, i.e.
+    #   the age a message must have before processing it.
+    # @param logger [Logger] The logger used for logging debug and error
+    #   messages to.
+
+    def initialize(stream_name:, delay:, logger: Logger.new("/dev/null"))
+      @stream_name = stream_name
+      @delay = delay
+      @logger = logger
+
+      @consumer = Consumer.new(name: "delayer", stream_name: "#{stream_name}.delay", logger: logger)
+      @batch = []
+    end
+
+    # Loops and blocks forever processing delay messages read from a delay
+    # stream.
+
+    def run
+      loop { run_once }
+    end
+
+    # Reads and processes a single batch of delay messages from a delay
+    # stream. You usually want to use the #run method instead, which
+    # loops/blocks forever.
+
+    def run_once
+      @consumer.run_once do |messages|
+        messages.each do |message|
+          seconds_to_sleep = message.message_id.to_f / 1_000 + @delay.to_f - Time.now.to_f
+
+          if seconds_to_sleep > 0
+            if @batch.size > 0
+              id = @batch.last.message_id
+
+              deliver
+
+              @consumer.commit id
+            end
+
+            sleep(seconds_to_sleep + 1)
+          end
+
+          @batch << message
+        end
+
+        deliver
+      end
+    rescue => e
+      @logger.error e
+
+      sleep 5
+
+      retry
+    end
+
+    private
+
+    def deliver
+      return if @batch.size.zero?
+
+      @logger.debug "Delayed #{@batch.size} messages for #{@delay.to_f} seconds on stream #{@stream_name}"
+
+      Redstream.connection_pool.with do |redis|
+        redis.pipelined do
+          @batch.each do |message|
+            redis.xadd Redstream.stream_key_name(@stream_name), payload: message.fields["payload"]
+          end
+        end
+
+        redis.xdel Redstream.stream_key_name("#{@stream_name}.delay"), @batch.map(&:message_id)
+      end
+
+      @batch = []
+    end
+  end
+end
+

data/lib/redstream/lock.rb

@@ -0,0 +1,80 @@
+
+require "securerandom"
+
+module Redstream
+  # @api private
+  #
+  # As the name suggests, the Redstream::Lock class implements a redis based
+  # locking mechanism. It atomically (lua script) gets/sets the lock key and
+  # updates its expire timeout, in case it currently holds the lock. Moreover,
+  # once it got the lock, it tries to keep it by updating the lock expire
+  # timeout from within a thread every 3 seconds.
+  #
+  # @example
+  #   lock = Redstream::Lock.new(name: "user_stream_lock")
+  #
+  #   loop do
+  #     got_lock = lock.acquire do
+  #       # ...
+  #     end
+  #
+  #     sleep(5) unless got_lock
+  #   end
+
+  class Lock
+    def initialize(name:)
+      @name = name
+      @id = SecureRandom.hex
+    end
+
+    def acquire(&block)
+      got_lock = get_lock
+      keep_lock(&block) if got_lock
+      got_lock
+    end
+
+    private
+
+    def keep_lock(&block)
+      stop = false
+      mutex = Mutex.new
+
+      Thread.new do
+        until mutex.synchronize { stop }
+          Redstream.connection_pool.with { |redis| redis.expire(Redstream.lock_key_name(@name), 5) }
+
+          sleep 3
+        end
+      end
+
+      block.call
+    ensure
+      mutex.synchronize do
+        stop = true
+      end
+    end
+
+    def get_lock
+      @get_lock_script =<<-EOF
+        local lock_key_name, id = ARGV[1], ARGV[2]
+
+        local cur = redis.call('get', lock_key_name)
+
+        if not cur then
+          redis.call('setex', lock_key_name, 5, id)
+
+          return true
+        elseif cur == id then
+          redis.call('expire', lock_key_name, 5)
+
+          return true
+        end
+
+        return false
+      EOF
+
+      Redstream.connection_pool.with { |redis| redis.eval(@get_lock_script, argv: [Redstream.lock_key_name(@name), @id]) }
+    end
+  end
+end
+

data/lib/redstream/message.rb

@@ -0,0 +1,52 @@
+
+module Redstream
+  # The Redstream::Message class wraps a raw redis stream message to allow hash
+  # and id/offset access as well as convenient parsing of the json payload.
+
+  class Message
+    # @api private
+    #
+    # Initializes the message.
+    #
+    # @param raw_message [Array] The raw message as returned by redis
+
+    def initialize(raw_message)
+      @message_id = raw_message[0]
+      @raw_message = raw_message
+    end
+
+    # Returns the message id, i.e. the redis message id consisting of a
+    # timestamp plus sequence number.
+    #
+    # @returns [String] The message id
+
+    def message_id
+      @message_id
+    end
+
+    # Returns the parsed message payload as provided by the model's
+    # #redstream_payload method. Check out Redstream::Model for more details.
+    #
+    # @return [Hash] The parsed payload
+
+    def payload
+      @payload ||= JSON.parse(fields["payload"])
+    end
+
+    # As a redis stream message allows to specify fields,
+    # this allows to retrieve the fields as a hash.
+    #
+    # @returns The fields hash
+
+    def fields
+      @fields ||= @raw_message[1]
+    end
+
+    # Returns the raw message content as returned by redis.
+
+    def raw_message
+      @raw_message
+    end
+  end
+end
+

data/lib/redstream/model.rb

@@ -0,0 +1,57 @@
+
+module Redstream
+  # Include Redstream::Model in your model to stream the model's updates via
+  # redis streams.
+  #
+  # @example
+  #   class User < ActiveRecord::Base
+  #     include Redstream::Model
+  #
+  #     # ...
+  #
+  #     redstream_callbacks
+  #   end
+
+  module Model
+    def self.included(base)
+      base.extend(ClassMethods)
+    end 
+
+    module ClassMethods
+      # Adds after_save, after_touch, after_destroy and, most importantly,
+      # after_commit callbacks. after_save, after_touch and after_destroy write
+      # a delay message to a delay stream. The delay messages are exactly like
+      # other messages, but will be read and replayed by a Redstream::Delayer
+      # only after a certain amount of time has passed (5 minutes usually) to
+      # fix potential inconsistencies which result from network or other issues
+      # in between database commit and the rails after_commit callback.
+      #
+      # @param producer [Redstream::Producer] A Redstream::Producer that is
+      #   responsible for writing to a redis stream
+
+      def redstream_callbacks(producer: Producer.new)
+        after_save { |object| producer.delay object }
+        after_touch { |object| producer.delay object }
+        after_destroy { |object| producer.delay object }
+        after_commit { |object| producer.queue object }
+      end 
+
+      def redstream_name
+        name.pluralize.underscore
+      end
+    end
+
+    # Override to customize the message payload. By default, the payload
+    # consists of the record id only (see example 1).
+    #
+    # @example Default
+    #   def redstream_payload
+    #     { id: id }
+    #   end
+
+    def redstream_payload
+      { id: id }
+    end
+  end 
+end
+

data/lib/redstream/producer.rb

@@ -0,0 +1,145 @@
+
+module Redstream
+  # A Redstream::Producer is responsible for writing the actual messages to
+  # redis. This includes the delay messages as well as the messages for
+  # immediate retrieval. Usually, you don't have to use a producer directly.
+  # Instead, Redstream::Model handles all producer related interaction.
+  # However, Redstream::Model is not able to recognize model updates resulting
+  # from model updates via e.g. #update_all, #delete_all, etc, i.e. updates
+  # which by-pass model callbacks. Thus, calls to e.g. #update_all must be
+  # wrapped with `find_in_batches` and Redstream::Producer#bulk (see example),
+  # to write these updates to the redis streams as well.
+  #
+  # @example
+  #   producer = Redstream::Producer.new
+  #
+  #   User.where(confirmed: true).find_in_batches do |users|
+  #     producer.bulk users do
+  #       User.where(id: users.map(&:id)).update_all(send_mailing: true)
+  #     end
+  #   end
+
+  class Producer
+    include MonitorMixin
+
+    # Initializes a new producer. In case you're using a distributed redis
+    # setup, you can use redis WAIT to improve real world data safety via the
+    # wait param.
+    #
+    # @param wait [Boolean, Integer] Defaults to false. Specify an integer to
+    #   enable using redis WAIT for writing delay messages. Check out the
+    #   redis docs for more info regarding WAIT.
+
+    def initialize(wait: false)
+      @wait = wait
+      @stream_name_cache = {}
+
+      super()
+    end
+
+    # Use to wrap calls to #update_all, #delete_all, etc. I.e. methods, which
+    # by-pass model lifecycle callbacks (after_save, etc.), as Redstream::Model
+    # can't recognize these updates and write them to redis streams
+    # automatically. You need to pass the records to be updated to the bulk
+    # method. The bulk method writes delay messages for the records to kafka,
+    # then yields and the writes the message for immediate retrieval. The
+    # method must ensure that the same set of records is used for the delay
+    # messages and the instant messages. Thus, you optimally, pass an array of
+    # records to it. If you pass an ActiveRecord::Relation, the method
+    # converts it to an array, i.e. loading all matching records into memory.
+    #
+    # @param records [#to_a] The object/objects that will be updated or deleted
+
+    def bulk(records)
+      records_array = Array(records)
+
+      bulk_delay(records_array)
+
+      yield
+
+      bulk_queue(records_array)
+    end
+
+    # @api private
+    #
+    # Writes delay messages to a delay stream in redis.
+    #
+    # @param records [#to_a] The object/objects that will be updated or deleted
+
+    def bulk_delay(records)
+      records.each_slice(250) do |slice|
+        Redstream.connection_pool.with do |redis|
+          redis.pipelined do
+            slice.map do |object|
+              redis.xadd Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload)
+            end
+          end
+        end
+      end
+
+      Redstream.connection_pool.with do |redis|
+        redis.wait(@wait, 0) if @wait
+      end
+
+      true
+    end
+
+    # @api private
+    #
+    # Writes messages to a stream in redis for immediate retrieval.
+    #
+    # @param records [#to_a] The object/objects that will be updated deleted
+
+    def bulk_queue(records)
+      records.each_slice(250) do |slice|
+        Redstream.connection_pool.with do |redis|
+          redis.pipelined do
+            slice.each do |object|
+              redis.xadd Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload)
+            end
+          end
+        end
+      end
+
+      true
+    end
+
+    # @api private
+    #
+    # Writes a single delay message to a delay stream in redis.
+    #
+    # @param object The object hat will be updated, deleted, etc.
+
+    def delay(object)
+      Redstream.connection_pool.with do |redis|
+        redis.xadd Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload)
+        redis.wait(@wait, 0) if @wait
+      end
+
+      true
+    end
+
+    # @api private
+    #
+    # Writes a single message to a stream in redis for immediate retrieval.
+    #
+    # @param object The object hat will be updated, deleted, etc.
+
+    def queue(object)
+      Redstream.connection_pool.with do |redis|
+        redis.xadd Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload)
+      end
+
+      true
+    end
+
+    private
+
+    def stream_name(object)
+      synchronize do
+        @stream_name_cache[object.class] ||= object.class.redstream_name
+      end
+    end
+  end
+end
+