redstream 0.0.1

data/lib/redstream/trimmer.rb

@@ -0,0 +1,91 @@
+
+module Redstream
+  # The Redstream::Trimmer class is neccessary to clean up messsages after all
+  # consumers have successfully processed and committed them. Otherwise they
+  # would fill up redis and finally bring redis down due to out of memory
+  # issues. The Trimmer will sleep for the specified interval in case there is
+  # nothing to trim. Please note that you must pass an array containing all
+  # consumer names reading from the stream which is about to be trimmed.
+  # Otherwise the Trimmer could trim messages from the stream before all
+  # consumers received the respective messages.
+  #
+  # @example
+  #   trimmer = Redstream::Trimmer.new(
+  #     interval: 30,
+  #     stream_name: "users",
+  #     consumer_names: ["indexer", "cacher"]
+  #   )
+  #
+  #   trimmer.run
+
+  class Trimmer
+    # Initializes a new trimmer. Accepts an interval to sleep for in case there
+    # is nothing to trim, the actual stream name, the consumer names as well as
+    # a logger for debug log messages.
+    #
+    # @param interval [Fixnum, Float] Specifies a time to sleep in case there is
+    #   nothing to trim.
+    # @param stream_name [String] The name of the stream that should be trimmed.
+    #   Please note, that redstream adds a prefix to the redis keys. However,
+    #   the stream_name param must be specified without any prefixes here. When
+    #   using Redstream::Model, the stream name is the downcased, pluralized
+    #   and underscored version of the model name. I.e., the stream name for a
+    #   'User' model will be 'users'
+    # @params consumer_names [Array] The list of all consumers reading from the
+    #   specified stream
+    # @param logger [Logger] A logger used for debug messages
+
+    def initialize(interval:, stream_name:, consumer_names:, logger: Logger.new("/dev/null"))
+      @interval = interval
+      @stream_name = stream_name
+      @consumer_names = consumer_names
+      @logger = logger
+      @lock = Lock.new(name: "trimmer:#{stream_name}")
+    end
+
+    # Loops and blocks forever trimming messages from the specified redis
+    # stream.
+
+    def run
+      loop { run_once }
+    end
+
+    # Runs the trimming a single time. You usually want to use the #run method
+    # instead, which loops/blocks forever.
+
+    def run_once
+      got_lock = @lock.acquire do
+        min_committed_id = Redstream.connection_pool.with do |redis|
+          offset_key_names = @consumer_names.map do |consumer_name|
+            Redstream.offset_key_name(stream_name: @stream_name, consumer_name: consumer_name)
+          end
+
+          redis.mget(offset_key_names).map(&:to_s).reject(&:empty?).min
+        end
+
+        return sleep(@interval) unless min_committed_id
+
+        loop do
+          messages = Redstream.connection_pool.with do |redis|
+            redis.xrange(Redstream.stream_key_name(@stream_name), "-", min_committed_id, count: 1_000)
+          end
+
+          return sleep(@interval) if messages.nil? || messages.empty?
+
+          Redstream.connection_pool.with { |redis| redis.xdel Redstream.stream_key_name(@stream_name), messages.map(&:first) }
+
+          @logger.debug "Trimmed #{messages.size} messages from #{@stream_name}"
+        end
+      end
+
+      sleep(5) unless got_lock
+    rescue => e
+      @logger.error e
+
+      sleep 5
+
+      retry
+    end
+  end
+end
+

data/lib/redstream/version.rb

@@ -0,0 +1,5 @@
+
+module Redstream
+  VERSION = "0.0.1"
+end
+

data/redstream.gemspec

@@ -0,0 +1,38 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'redstream/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "redstream"
+  spec.version       = Redstream::VERSION
+  spec.authors       = ["Benjamin Vetter"]
+  spec.email         = ["vetter@plainpicture.de"]
+  spec.summary       = %q{Using redis streams to keep your primary database in sync with secondary datastores}
+  spec.description   = %q{Using redis streams to keep your primary database in sync with secondary datastores}
+  spec.homepage      = "https://github.com/mrkamel/redstream"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "database_cleaner"
+  spec.add_development_dependency "sqlite3", "1.3.13"
+  spec.add_development_dependency "factory_bot"
+  spec.add_development_dependency "timecop"
+  spec.add_development_dependency "concurrent-ruby"
+  spec.add_development_dependency "rspec-instafail"
+  spec.add_development_dependency "mocha"
+
+  spec.add_dependency "connection_pool"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "redis", ">= 4.1.0"
+  spec.add_dependency "json"
+end
+

data/spec/redstream/consumer_spec.rb

@@ -0,0 +1,90 @@
+
+require File.expand_path("../spec_helper", __dir__)
+
+RSpec.describe Redstream::Consumer do
+  describe "#run_once" do
+    it "doesn't call the block without messages" do
+      called = false
+
+      Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 5).run_once do |batch|
+        called = true
+      end
+
+      expect(called).to eq(false)
+    end
+
+    it "is mutually exclusive" do
+      create :product
+
+      calls = Concurrent::AtomicFixnum.new(0)
+
+      threads = Array.new(2) do |i|
+        Thread.new do
+          Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 5).run_once do |batch|
+            calls.increment
+
+            sleep 1
+          end
+        end
+      end
+
+      threads.each(&:join)
+
+      expect(calls.value).to eq(1)
+    end
+
+    it "is using the existing offset" do
+      create_list(:product, 2)
+
+      all_messages = redis.xrange(Redstream.stream_key_name("products"), "-", "+")
+
+      expect(all_messages.size).to eq(2)
+
+      redis.set(Redstream.offset_key_name(stream_name: "products", consumer_name: "consumer"), all_messages.first[0])
+
+      messages = nil
+
+      consumer = Redstream::Consumer.new(name: "consumer", stream_name: "products")
+
+      consumer.run_once do |batch|
+        messages = batch
+      end
+
+      expect(messages.size).to eq(1)
+      expect(messages.first.raw_message).to eq(all_messages.last)
+    end
+
+    it "yields messages in batches" do
+      products = create_list(:product, 15)
+
+      consumer = Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 10)
+
+      messages = nil
+
+      consumer.run_once do |batch|
+        messages = batch
+      end
+
+      expect(messages.size).to eq(10)
+
+      consumer.run_once do |batch|
+        messages = batch
+      end
+
+      expect(messages.size).to eq(5)
+    end
+
+    it "updates the offset" do
+      create :product
+
+      expect(redis.get(Redstream.offset_key_name(stream_name: "products", consumer_name: "consumer"))).to be(nil)
+
+      all_messages = redis.xrange(Redstream.stream_key_name("products"), "-", "+")
+
+      Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once {}
+
+      expect(redis.get(Redstream.offset_key_name(stream_name: "products", consumer_name: "consumer"))).to eq(all_messages.last[0])
+    end
+  end
+end
+

data/spec/redstream/delayer_spec.rb

@@ -0,0 +1,53 @@
+
+require File.expand_path("../spec_helper", __dir__)
+
+RSpec.describe Redstream::Delayer do
+  describe "#run_once" do
+    it "copies expired messages to their target streams" do
+      redis.xadd Redstream.stream_key_name("target.delay"), payload: JSON.dump(value: "message")
+
+      expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(0)
+
+      Redstream::Delayer.new(stream_name: "target", delay: 0).run_once
+
+      expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(1)
+      expect(redis.xrange(Redstream.stream_key_name("target")).last[1]).to eq("payload" => JSON.dump(value: "message"))
+    end
+
+    it "delivers and commit before falling asleep" do
+      redis.xadd Redstream.stream_key_name("target.delay"), payload: JSON.dump(value: "message")
+      sleep 3
+      redis.xadd Redstream.stream_key_name("target.delay"), payload: JSON.dump(value: "message")
+
+      thread = Thread.new do
+        Redstream::Delayer.new(stream_name: "target", delay: 1).run_once
+      end
+
+      sleep 1
+
+      expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(1)
+      expect(redis.get(Redstream.offset_key_name(stream_name: "target.delay", consumer_name: "delayer"))).not_to be_nil
+
+      thread.join
+
+      expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(2)
+    end
+
+    it "does not copy not yet expired messages" do
+      redis.xadd Redstream.stream_key_name("target.delay"), payload: JSON.dump(value: "message")
+
+      thread = Thread.new do
+        Redstream::Delayer.new(stream_name: "target", delay: 2).run_once
+      end
+
+      sleep 1
+
+      expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(0)
+
+      thread.join
+
+      expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(1)
+    end
+  end
+end
+

data/spec/redstream/lock_spec.rb

@@ -0,0 +1,68 @@
+
+require File.expand_path("../spec_helper", __dir__)
+
+RSpec.describe Redstream::Lock do
+  describe "#acquire" do
+    it "gets a lock" do
+      lock_results = Concurrent::Array.new
+      calls = Concurrent::AtomicFixnum.new(0)
+
+      threads = Array.new(2) do |i|
+        Thread.new do
+          lock_results << Redstream::Lock.new(name: "lock").acquire do
+            calls.increment
+
+            sleep 1
+          end
+        end
+      end
+
+      threads.each(&:join)
+
+      expect(calls.value).to eq(1)
+      expect(lock_results.to_set).to eq([1, nil].to_set)
+    end
+
+    it "keeps the lock" do
+      threads = []
+      calls = Concurrent::Array.new
+
+      threads << Thread.new do
+        Redstream::Lock.new(name: "lock").acquire do
+          calls << "thread-1"
+
+          sleep 6
+        end
+      end
+
+      sleep 6
+
+      threads << Thread.new do
+        Redstream::Lock.new(name: "lock").acquire do
+          calls << "thread-2"
+        end
+      end
+
+      threads.each(&:join)
+
+      expect(calls).to eq(["thread-1"])
+    end
+
+    it "does not lock itself" do
+      lock = Redstream::Lock.new(name: "lock")
+
+      lock_results = []
+      calls = 0
+
+      2.times do
+        lock_results << lock.acquire do
+          calls += 1
+        end
+      end
+
+      expect(calls).to eq(2)
+      expect(lock_results).to eq([1, 1])
+    end
+  end
+end
+

data/spec/redstream/model_spec.rb

@@ -0,0 +1,57 @@
+
+require File.expand_path("../spec_helper", __dir__)
+
+RSpec.describe Redstream::Model do
+  it "adds a delay message after save" do
+    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+
+    time = Time.now
+
+    product = Timecop.freeze(time) do
+      create(:product)
+    end
+
+    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(1)
+    expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+  end
+
+  it "adds a delay message after touch" do
+    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+
+    product = create(:product)
+
+    time = Time.now
+
+    Timecop.freeze(time) do
+      product.touch
+    end
+
+    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(2)
+    expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+  end
+
+  it "adds a delay message after destroy" do
+    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+
+    product = create(:product)
+
+    time = Time.now
+
+    Timecop.freeze(time) do
+      product.touch
+    end
+
+    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(2)
+    expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+  end
+
+  it "adds a queue message after commit" do
+    expect(redis.xlen(Redstream.stream_key_name("products"))).to eq(0)
+
+    product = create(:product)
+
+    expect(redis.xlen(Redstream.stream_key_name("products"))).to eq(1)
+    expect(redis.xrange(Redstream.stream_key_name("products"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+  end
+end
+

data/spec/redstream/producer_spec.rb

@@ -0,0 +1,79 @@
+
+require File.expand_path("../spec_helper", __dir__)
+
+RSpec.describe Redstream::Producer do
+  describe "#queue" do
+    it "adds a queue message for individual objects" do
+      product = create(:product)
+
+      stream_key_name = Redstream.stream_key_name("products")
+
+      expect { Redstream::Producer.new.queue(product) }.to change { redis.xlen(stream_key_name) }.by(1)
+      expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+    end
+  end
+
+  describe "#delay" do
+    it "adds a delay message for individual objects" do
+      product = create(:product)
+
+      stream_key_name = Redstream.stream_key_name("products.delay")
+
+      expect { Redstream::Producer.new.delay(product) }.to change { redis.xlen(stream_key_name) }.by(1)
+      expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+    end
+
+    it "resepects wait" do
+      product = create(:product)
+
+      stream_key_name = Redstream.stream_key_name("products.delay")
+
+      expect { Redstream::Producer.new(wait: 0).delay(product) }.to change { redis.xlen(stream_key_name) }.by(1)
+    end
+  end
+
+  describe "#bulk_queue" do
+    it "adds bulk queue messages for scopes" do
+      products = create_list(:product, 2)
+
+      stream_key_name = Redstream.stream_key_name("products")
+
+      expect { Redstream::Producer.new.bulk_queue(Product.all) }.to change { redis.xlen(stream_key_name) }.by(2)
+
+      messages = redis.xrange(stream_key_name, "-", "+").last(2).map { |message| message[1] }
+
+      expect(messages).to eq([
+        { "payload" => JSON.dump(products[0].redstream_payload) },
+        { "payload" => JSON.dump(products[1].redstream_payload) }
+      ])
+    end
+  end
+
+  describe "#bulk_delay" do
+    it "adds bulk delay messages for scopes" do
+      products = create_list(:product, 2)
+
+      stream_key_name = Redstream.stream_key_name("products.delay")
+
+      expect { Redstream::Producer.new.bulk_delay(Product.all) }.to change { redis.xlen(stream_key_name) }.by(2)
+
+      messages = redis.xrange(stream_key_name, "-", "+").last(2).map { |message| message[1] }
+
+      expect(messages).to eq([
+        { "payload" => JSON.dump(products[0].redstream_payload) },
+        { "payload" => JSON.dump(products[1].redstream_payload) }
+      ])
+    end
+
+    it "should resepect wait for delay" do
+      product = create(:product)
+
+      stream_key_name = Redstream.stream_key_name("products.delay")
+
+      products = create_list(:product, 2)
+
+      expect { Redstream::Producer.new(wait: 0).bulk_delay(products) }.to change { redis.xlen(stream_key_name) }.by(2)
+    end
+  end
+end
+