redstream 0.0.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e1ddc2700836c469d1ca61069e3416c21e657e05725b92e75969aa8110768e3
-  data.tar.gz: c8565f3754b3fd4f66823d6d7035e814f73e27abdb15936ecd906f5f07dd8643
+  metadata.gz: 160386bd218415c11c556b1cdfc8824ed347e7279fac67f390c07b37c7128eff
+  data.tar.gz: '087e4ecaee15bd2f9db4367f53297a72f01c8de1cc093ae25ff3d4eb8ae29b90'
 SHA512:
-  metadata.gz: edd496df8d06b98b9318b9796f400e2c0870edfc84c3aa7f9c7946dbe6cf91c5a8c0ab32425d627bc20c585389eab92ed1b290e57e0df856e8995547d8a9b7c6
-  data.tar.gz: 4893d2197f427479e4df0821ca29a23ee98a604fa73680f955da8d2c71cbdb192d006c476dc3bd6c03c719da327c3d9b6f207842082a64133f0fe2383771aef5
+  metadata.gz: d4d0945a2d2cb4b38dd398e5368963b5c8ba63aa93bc065bcb0c78a0e6bd7d2c0a7f825dd76b7b1d0ff79b6874bf1c3d0cb8f29843977d7676d341cfa584ed5d
+  data.tar.gz: b53bc33ce21244d64307b22f98185367e7c5806aeed9e50ed2b9dd123278b19236a049042bde26870dacd99702df49966674d1c629bf73ad955ff176f3f25a7b

data/.github/workflows/test.yml

@@ -0,0 +1,28 @@
+name: test
+on: [push, pull_request]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ['2.5', '2.6', '2.7']
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - uses: actions/cache@v1
+      id: cache
+      with:
+        path: vendor/bundler
+        key: ${{ hashFiles('Gemfile.lock') }}-${{ matrix.ruby }}
+    - run: |
+        gem install bundler
+        bundle install --path=vendor/bundler
+        bundle exec rspec
+        bundle exec rubocop

data/.rubocop.yml

@@ -0,0 +1,122 @@
+AllCops:
+  NewCops: enable
+
+Lint/AssignmentInCondition:
+  Enabled: false
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+Lint/AmbiguousBlockAssociation:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Lint/RedundantRequireStatement:
+  Enabled: false
+
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+Layout/FirstArrayElementIndentation:
+  EnforcedStyle: consistent
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+Style/SpecialGlobalVars:
+  EnforcedStyle: use_english_names
+
+Security/Eval:
+  Enabled: false
+
+Style/WordArray:
+  EnforcedStyle: brackets
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/TrivialAccessors:
+  Enabled: false
+
+Style/Alias:
+  Enabled: false
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Metrics/ClassLength:
+  Enabled: false
+
+Naming/MethodParameterName:
+  Enabled: false
+
+Style/SymbolArray:
+  EnforcedStyle: brackets
+
+Layout/RescueEnsureAlignment:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Style/ZeroLengthPredicate:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/BlockNesting:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Naming/AccessorMethodName:
+  Enabled: false
+
+Naming/MemoizedInstanceVariableName:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/Documentation:
+  Enabled: false
+
+Naming/ConstantName:
+  Enabled: false
+
+Style/MutableConstant:
+  Enabled: false
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
+Layout/ParameterAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+Lint/UnusedMethodArgument:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/RedundantBegin:
+  Enabled: false

data/.travis.yml

@@ -1,10 +1,13 @@
 sudo: false
 language: ruby
+cache: bundler
 rvm:
-  - ruby-head
+  - ruby-2.6.2
 before_install:
   - docker-compose up -d
   - sleep 10
 install:
   - travis_retry bundle install
-script: rspec
+script:
+  - rspec
+  - rubocop

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+# CHANGELOG
+
+## v0.4.0
+* Make delay message id params in queue methods optional
+
+## v0.3.0
+* Pipeline deletion of delay messages
+
+## v0.2.0
+* Delete delay messages after queue messages are sent
+
+## v0.1.1
+* Fix missing queue message in `after_commit on: :destroy`
+
+## v0.1.0
+* No longer queue/delay in `after_save`/`after_commit` if no changes occurred
+* Added `Redstream.stream_size`

data/Gemfile

@@ -1,5 +1,3 @@
-
 source "https://rubygems.org"
 
 gemspec
-

data/README.md

@@ -4,7 +4,7 @@
 **Using redis streams to keep your primary database in sync with secondary
 datastores (e.g. elasticsearch).**
 
-[![Build Status](https://secure.travis-ci.org/mrkamel/redstream.png?branch=master)](http://travis-ci.org/mrkamel/redstream)
+[![Build Status](https://github.com/mrkamel/redstream/workflows/test/badge.svg?branch=master)](https://github.com/mrkamel/redstream/actions?query=workflow%3Atest)
 
 ## Installation
 
@@ -86,9 +86,9 @@ end
 More concretely, `after_save`, `after_touch` and `after_destroy` only write
 "delay" messages to an additional redis stream. Delay message are like any
 other messages, but they get processed by a `Redstream::Delayer` and the
-`Delayer`will wait for some (configurable) delay/time before processing them.
+`Delayer` will wait for some (configurable) delay/time before processing them.
 As the `Delayer` is neccessary to fix inconsistencies, the delay must be at
-least as long as your maxiumum database transaction time. Contrary,
+least as long as your maximum database transaction time. Contrary,
 `after_commit` writes messages to a redis stream from which the messages can
 be fetched immediately to keep the secondary datastores updated in
 near-realtime. The reasoning of all this is simple: usually, i.e. by using only
@@ -97,7 +97,9 @@ any errors occurring in between `after_save` and `after_commit` result in
 inconsistencies between your primary and secondary datastore. By using these
 kinds of "delay" messages triggered by `after_save` and fetched after e.g. 5
 minutes, errors occurring in between `after_save` and `after_commit` can be
-fixed when the delay message get processed.
+fixed when the delay message get processed. Please note that redstream deletes
+delay messages after the messages for immediate retrieval have been
+successfully sent, such that messages will not be processed twice, usually.
 
 Any messages are fetched in batches, such that e.g. elasticsearch can be
 updated using its bulk API. For instance, depending on which elasticsearch ruby

data/Rakefile

@@ -6,4 +6,3 @@ Rake::TestTask.new(:test) do |t|
   t.pattern = "test/**/*_test.rb"
   t.verbose = true
 end
-

data/lib/redstream.rb

@@ -1,4 +1,3 @@
-
 require "active_support/inflector"
 require "connection_pool"
 require "redis"
@@ -59,6 +58,17 @@ module Redstream
     @namespace
   end
 
+  # Returns the length of the specified stream.
+  #
+  # @param stream_name [String] The stream name
+  # @return [Integer] The length of the stream
+
+  def self.stream_size(stream_name)
+    connection_pool.with do |redis|
+      redis.xlen(stream_key_name(stream_name))
+    end
+  end
+
   # Returns the max id of the specified stream, i.e. the id of the
   # last/newest message added. Returns nil for empty streams.
   #
@@ -69,7 +79,7 @@ module Redstream
     connection_pool.with do |redis|
       message = redis.xrevrange(stream_key_name(stream_name), "+", "-", count: 1).first
 
-      return unless message
+      return nil unless message
 
       message[0]
     end
@@ -131,4 +141,3 @@ module Redstream
     [namespace, "redstream"].compact.join(":")
   end
 end
-

data/lib/redstream/consumer.rb

@@ -1,4 +1,3 @@
-
 require "thread"
 
 module Redstream
@@ -92,7 +91,7 @@ module Redstream
       end
 
       sleep(5) unless got_lock
-    rescue => e
+    rescue StandardError => e
       @logger.error e
 
       sleep 5
@@ -112,4 +111,3 @@ module Redstream
     end
   end
 end
-

data/lib/redstream/delayer.rb

@@ -1,4 +1,3 @@
-
 module Redstream
   # The Redstream::Delayer class is responsible for reading messages from
   # special delay streams which are used to fix inconsistencies resulting from
@@ -68,7 +67,7 @@ module Redstream
 
         deliver
       end
-    rescue => e
+    rescue StandardError => e
       @logger.error e
 
       sleep 5
@@ -97,4 +96,3 @@ module Redstream
     end
   end
 end
-

data/lib/redstream/lock.rb

@@ -1,4 +1,3 @@
-
 require "securerandom"
 
 module Redstream
@@ -55,7 +54,7 @@ module Redstream
     end
 
     def get_lock
-      @get_lock_script =<<-EOF
+      @get_lock_script = <<~GET_LOCK_SCRIPT
         local lock_key_name, id = ARGV[1], ARGV[2]
 
         local cur = redis.call('get', lock_key_name)
@@ -71,10 +70,9 @@ module Redstream
         end
 
         return false
-      EOF
+      GET_LOCK_SCRIPT
 
       Redstream.connection_pool.with { |redis| redis.eval(@get_lock_script, argv: [Redstream.lock_key_name(@name), @id]) }
     end
   end
 end
-

data/lib/redstream/message.rb

@@ -1,4 +1,3 @@
-
 module Redstream
   # The Redstream::Message class wraps a raw redis stream message to allow hash
   # and id/offset access as well as convenient parsing of the json payload.
@@ -49,4 +48,3 @@ module Redstream
     end
   end
 end
-

data/lib/redstream/model.rb

@@ -1,4 +1,3 @@
-
 module Redstream
   # Include Redstream::Model in your model to stream the model's updates via
   # redis streams.
@@ -13,9 +12,11 @@ module Redstream
   #   end
 
   module Model
+    IVAR_DELAY_MESSAGE_ID = :@__redstream_delay_message_id__
+
     def self.included(base)
       base.extend(ClassMethods)
-    end 
+    end
 
     module ClassMethods
       # Adds after_save, after_touch, after_destroy and, most importantly,
@@ -30,11 +31,22 @@ module Redstream
       #   responsible for writing to a redis stream
 
       def redstream_callbacks(producer: Producer.new)
-        after_save { |object| producer.delay object }
-        after_touch { |object| producer.delay object }
-        after_destroy { |object| producer.delay object }
-        after_commit { |object| producer.queue object }
-      end 
+        after_save    { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) if object.saved_changes.present? }
+        after_touch   { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
+        after_destroy { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
+
+        after_commit(on: [:create, :update]) do |object|
+          if object.saved_changes.present?
+            producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
+            instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
+          end
+        end
+
+        after_commit(on: :destroy) do |object|
+          producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
+          instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
+        end
+      end
 
       def redstream_name
         name.pluralize.underscore
@@ -52,6 +64,5 @@ module Redstream
     def redstream_payload
       { id: id }
     end
-  end 
+  end
 end
-

data/lib/redstream/producer.rb

@@ -1,4 +1,3 @@
-
 module Redstream
   # A Redstream::Producer is responsible for writing the actual messages to
   # redis. This includes the delay messages as well as the messages for
@@ -53,11 +52,11 @@ module Redstream
     def bulk(records)
       records_array = Array(records)
 
-      bulk_delay(records_array)
+      delay_message_ids = bulk_delay(records_array)
 
       yield
 
-      bulk_queue(records_array)
+      bulk_queue(records_array, delay_message_ids: delay_message_ids)
     end
 
     # @api private
@@ -65,13 +64,15 @@ module Redstream
     # Writes delay messages to a delay stream in redis.
     #
     # @param records [#to_a] The object/objects that will be updated or deleted
+    #
+    # @return The redis message ids
 
     def bulk_delay(records)
-      records.each_slice(250) do |slice|
+      res = records.each_slice(250).flat_map do |slice|
         Redstream.connection_pool.with do |redis|
           redis.pipelined do
-            slice.map do |object|
-              redis.xadd Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload)
+            slice.each do |object|
+              redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload))
             end
           end
         end
@@ -81,7 +82,7 @@ module Redstream
         redis.wait(@wait, 0) if @wait
       end
 
-      true
+      res
     end
 
     # @api private
@@ -89,13 +90,15 @@ module Redstream
     # Writes messages to a stream in redis for immediate retrieval.
     #
     # @param records [#to_a] The object/objects that will be updated deleted
+    # @param delay_message_ids [#to_a] The delay message ids to delete
 
-    def bulk_queue(records)
-      records.each_slice(250) do |slice|
+    def bulk_queue(records, delay_message_ids: nil)
+      records.each_with_index.each_slice(250) do |slice|
         Redstream.connection_pool.with do |redis|
           redis.pipelined do
-            slice.each do |object|
-              redis.xadd Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload)
+            slice.each do |object, index|
+              redis.xadd(Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload))
+              redis.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_ids[index]) if delay_message_ids
             end
           end
         end
@@ -108,15 +111,16 @@ module Redstream
     #
     # Writes a single delay message to a delay stream in redis.
     #
-    # @param object The object hat will be updated, deleted, etc.
+    # @param object The object that will be updated, deleted, etc.
+    #
+    # @return The redis message id
 
     def delay(object)
       Redstream.connection_pool.with do |redis|
-        redis.xadd Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload)
+        res = redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload))
         redis.wait(@wait, 0) if @wait
+        res
       end
-
-      true
     end
 
     # @api private
@@ -124,10 +128,14 @@ module Redstream
     # Writes a single message to a stream in redis for immediate retrieval.
     #
     # @param object The object hat will be updated, deleted, etc.
+    # @param delay_message_id The delay message id to delete
 
-    def queue(object)
+    def queue(object, delay_message_id: nil)
       Redstream.connection_pool.with do |redis|
-        redis.xadd Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload)
+        redis.pipelined do
+          redis.xadd(Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload))
+          redis.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_id) if delay_message_id
+        end
       end
 
       true
@@ -142,4 +150,3 @@ module Redstream
     end
   end
 end
-

data/lib/redstream/trimmer.rb

@@ -1,4 +1,3 @@
-
 module Redstream
   # The Redstream::Trimmer class is neccessary to clean up messsages after all
   # consumers have successfully processed and committed them. Otherwise they
@@ -79,7 +78,7 @@ module Redstream
       end
 
       sleep(5) unless got_lock
-    rescue => e
+    rescue StandardError => e
       @logger.error e
 
       sleep 5
@@ -88,4 +87,3 @@ module Redstream
     end
   end
 end
-

data/lib/redstream/version.rb

@@ -1,5 +1,3 @@
-
 module Redstream
-  VERSION = "0.0.1"
+  VERSION = "0.4.0"
 end
-

data/redstream.gemspec

@@ -1,15 +1,14 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'redstream/version'
+require "redstream/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "redstream"
   spec.version       = Redstream::VERSION
   spec.authors       = ["Benjamin Vetter"]
   spec.email         = ["vetter@plainpicture.de"]
-  spec.summary       = %q{Using redis streams to keep your primary database in sync with secondary datastores}
-  spec.description   = %q{Using redis streams to keep your primary database in sync with secondary datastores}
+  spec.summary       = "Using redis streams to keep your primary database in sync with secondary datastores"
+  spec.description   = "Using redis streams to keep your primary database in sync with secondary datastores"
   spec.homepage      = "https://github.com/mrkamel/redstream"
   spec.license       = "MIT"
 
@@ -18,21 +17,21 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
   spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "concurrent-ruby"
   spec.add_development_dependency "database_cleaner"
-  spec.add_development_dependency "sqlite3", "1.3.13"
   spec.add_development_dependency "factory_bot"
-  spec.add_development_dependency "timecop"
-  spec.add_development_dependency "concurrent-ruby"
-  spec.add_development_dependency "rspec-instafail"
   spec.add_development_dependency "mocha"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-instafail"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "timecop"
 
-  spec.add_dependency "connection_pool"
   spec.add_dependency "activesupport"
-  spec.add_dependency "redis", ">= 4.1.0"
+  spec.add_dependency "connection_pool"
   spec.add_dependency "json"
+  spec.add_dependency "redis", ">= 4.1.0"
 end
-

data/spec/redstream/consumer_spec.rb

@@ -1,4 +1,3 @@
-
 require File.expand_path("../spec_helper", __dir__)
 
 RSpec.describe Redstream::Consumer do
@@ -6,7 +5,7 @@ RSpec.describe Redstream::Consumer do
     it "doesn't call the block without messages" do
       called = false
 
-      Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 5).run_once do |batch|
+      Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 5).run_once do |_batch|
         called = true
       end
 
@@ -18,9 +17,9 @@ RSpec.describe Redstream::Consumer do
 
       calls = Concurrent::AtomicFixnum.new(0)
 
-      threads = Array.new(2) do |i|
+      threads = Array.new(2) do |_i|
         Thread.new do
-          Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 5).run_once do |batch|
+          Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 5).run_once do |_batch|
             calls.increment
 
             sleep 1
@@ -55,7 +54,7 @@ RSpec.describe Redstream::Consumer do
     end
 
     it "yields messages in batches" do
-      products = create_list(:product, 15)
+      create_list(:product, 15)
 
       consumer = Redstream::Consumer.new(name: "consumer", stream_name: "products", batch_size: 10)
 
@@ -81,10 +80,11 @@ RSpec.describe Redstream::Consumer do
 
       all_messages = redis.xrange(Redstream.stream_key_name("products"), "-", "+")
 
-      Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once {}
+      Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do
+        # nothing
+      end
 
       expect(redis.get(Redstream.offset_key_name(stream_name: "products", consumer_name: "consumer"))).to eq(all_messages.last[0])
     end
   end
 end
-

data/spec/redstream/delayer_spec.rb

@@ -1,4 +1,3 @@
-
 require File.expand_path("../spec_helper", __dir__)
 
 RSpec.describe Redstream::Delayer do
@@ -50,4 +49,3 @@ RSpec.describe Redstream::Delayer do
     end
   end
 end
-

data/spec/redstream/lock_spec.rb

@@ -1,4 +1,3 @@
-
 require File.expand_path("../spec_helper", __dir__)
 
 RSpec.describe Redstream::Lock do
@@ -7,7 +6,7 @@ RSpec.describe Redstream::Lock do
       lock_results = Concurrent::Array.new
       calls = Concurrent::AtomicFixnum.new(0)
 
-      threads = Array.new(2) do |i|
+      threads = Array.new(2) do |_i|
         Thread.new do
           lock_results << Redstream::Lock.new(name: "lock").acquire do
             calls.increment
@@ -65,4 +64,3 @@ RSpec.describe Redstream::Lock do
     end
   end
 end
-

data/spec/redstream/model_spec.rb

@@ -1,57 +1,136 @@
-
 require File.expand_path("../spec_helper", __dir__)
 
 RSpec.describe Redstream::Model do
-  it "adds a delay message after save" do
-    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+  describe "after_save" do
+    it "adds a delay message after_save" do
+      Product.transaction do
+        expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      end
+    end
 
-    time = Time.now
+    it "assigns the delay message id" do
+      Product.transaction do
+        expect(create(:product).instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
+      end
+    end
 
-    product = Timecop.freeze(time) do
-      create(:product)
+    it "adds the correct payload for the delay message" do
+      Product.transaction do
+        product = create(:product)
+
+        expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      end
     end
 
-    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(1)
-    expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
-  end
+    it "adds a queue message after_save on commit" do
+      expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products")) }
+    end
 
-  it "adds a delay message after touch" do
-    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+    it "deletes the delay message on commit" do
+      product = create(:product)
 
-    product = create(:product)
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
+    end
 
-    time = Time.now
+    it "does not add a delay message after_save if there are no changes" do
+      product = create(:product)
 
-    Timecop.freeze(time) do
-      product.touch
+      expect { product.save }.not_to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
     end
 
-    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(2)
-    expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+    it "does not add a queue messages after_save on commit if there are no changes" do
+      product = create(:product)
+
+      expect { product.save }.not_to change { redis.xlen(Redstream.stream_key_name("products")) }
+    end
   end
 
-  it "adds a delay message after destroy" do
-    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+  describe "after_touch" do
+    it "adds a delay message after touch" do
+      product = create(:product)
 
-    product = create(:product)
+      Product.transaction do
+        expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      end
+    end
 
-    time = Time.now
+    it "assigns the delay message id" do
+      product = create(:product)
 
-    Timecop.freeze(time) do
-      product.touch
+      Product.transaction do
+        product.touch
+
+        expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
+      end
+    end
+
+    it "sets the correct payload for the delay message" do
+      product = create(:product)
+
+      Product.transaction do
+        product.touch
+
+        expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      end
+    end
+
+    it "adds a queue message after touch on commit" do
+      product = create(:product)
+
+      expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products")) }
     end
 
-    expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(2)
-    expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+    it "deletes the delay message after touch on commit" do
+      product = create(:product)
+      product.touch
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
+    end
   end
 
-  it "adds a queue message after commit" do
-    expect(redis.xlen(Redstream.stream_key_name("products"))).to eq(0)
+  describe "after_destroy" do
+    it "adds a delay message after destroy" do
+      product = create(:product)
+
+      Product.transaction do
+        expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      end
+    end
+
+    it "assigns the delay message id" do
+      product = create(:product)
+
+      Product.transaction do
+        product.destroy
+
+        expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
+      end
+    end
 
-    product = create(:product)
+    it "sets the correct payload for the delay message" do
+      product = create(:product)
 
-    expect(redis.xlen(Redstream.stream_key_name("products"))).to eq(1)
-    expect(redis.xrange(Redstream.stream_key_name("products"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      Product.transaction do
+        product.destroy
+
+        expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      end
+    end
+
+    it "adds a queue messages after destroy on commit" do
+      product = create(:product)
+
+      expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products")) }.by(1)
+    end
+
+    it "deletes the delay message after destroy on commit" do
+      product = create(:product)
+      product.destroy
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
+    end
   end
 end
-

data/spec/redstream/producer_spec.rb

@@ -1,4 +1,3 @@
-
 require File.expand_path("../spec_helper", __dir__)
 
 RSpec.describe Redstream::Producer do
@@ -11,6 +10,17 @@ RSpec.describe Redstream::Producer do
       expect { Redstream::Producer.new.queue(product) }.to change { redis.xlen(stream_key_name) }.by(1)
       expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
     end
+
+    it "deletes the delay message when given" do
+      product = create(:product)
+
+      producer = Redstream::Producer.new
+
+      id = producer.delay(product)
+      producer.queue(product, delay_message_id: id)
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+    end
   end
 
   describe "#delay" do
@@ -32,6 +42,57 @@ RSpec.describe Redstream::Producer do
     end
   end
 
+  describe "#bulk" do
+    it "adds bulk delay messages for scopes" do
+      products = create_list(:product, 2)
+
+      stream_key_name = Redstream.stream_key_name("products")
+
+      expect(redis.xlen("#{stream_key_name}.delay")).to eq(0)
+
+      Redstream::Producer.new.bulk(Product.all) do
+        messages = redis.xrange("#{stream_key_name}.delay", "-", "+").last(2).map { |message| message[1] }
+
+        expect(messages).to eq([
+          { "payload" => JSON.dump(products[0].redstream_payload) },
+          { "payload" => JSON.dump(products[1].redstream_payload) }
+        ])
+      end
+    end
+
+    it "adds bulk queue messages for scopes" do
+      products = create_list(:product, 2)
+
+      stream_key_name = Redstream.stream_key_name("products")
+
+      expect do
+        Redstream::Producer.new.bulk(Product.all) do
+          # nothing
+        end
+      end.to change { redis.xlen(stream_key_name) }.by(2)
+
+      messages = redis.xrange(stream_key_name, "-", "+").last(2).map { |message| message[1] }
+
+      expect(messages).to eq([
+        { "payload" => JSON.dump(products[0].redstream_payload) },
+        { "payload" => JSON.dump(products[1].redstream_payload) }
+      ])
+    end
+
+    it "deletes the delay messages after the queue messages have been sent" do
+      products = create_list(:product, 2)
+      producer = Redstream::Producer.new
+
+      other_delay_message_id = producer.delay(create(:product))
+
+      producer.bulk(products) do
+        expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(3)
+      end
+
+      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
+    end
+  end
+
   describe "#bulk_queue" do
     it "adds bulk queue messages for scopes" do
       products = create_list(:product, 2)
@@ -47,6 +108,18 @@ RSpec.describe Redstream::Producer do
         { "payload" => JSON.dump(products[1].redstream_payload) }
       ])
     end
+
+    it "deletes the delay messages after the queue messages have been sent" do
+      products = create_list(:product, 2)
+      producer = Redstream::Producer.new
+
+      delay_message_ids = producer.bulk_delay(products)
+      other_delay_message_id = producer.delay(create(:product))
+
+      producer.bulk_queue(products, delay_message_ids: delay_message_ids)
+
+      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
+    end
   end
 
   describe "#bulk_delay" do
@@ -65,8 +138,8 @@ RSpec.describe Redstream::Producer do
       ])
     end
 
-    it "should resepect wait for delay" do
-      product = create(:product)
+    it "should respect wait for delay" do
+      create(:product)
 
       stream_key_name = Redstream.stream_key_name("products.delay")
 
@@ -76,4 +149,3 @@ RSpec.describe Redstream::Producer do
     end
   end
 end
-

data/spec/redstream/trimmer_spec.rb

@@ -1,4 +1,3 @@
-
 require File.expand_path("../spec_helper", __dir__)
 
 RSpec.describe Redstream::Trimmer do
@@ -29,4 +28,3 @@ RSpec.describe Redstream::Trimmer do
     end
   end
 end
-

data/spec/redstream_spec.rb

@@ -1,4 +1,3 @@
-
 require File.expand_path("spec_helper", __dir__)
 
 RSpec.describe Redstream do
@@ -31,11 +30,21 @@ RSpec.describe Redstream do
     end
   end
 
+  describe ".stream_size" do
+    it "returns the stream's size" do
+      expect(Redstream.stream_size("products")).to eq(0)
+
+      redis.xadd("redstream:stream:products", key: "value")
+
+      expect(Redstream.stream_size("products")).to eq(1)
+    end
+  end
+
   describe ".max_stream_id" do
     it "returns the stream's max id" do
       expect(Redstream.max_stream_id("products")).to be_nil
 
-      id1 = redis.xadd("redstream:stream:products", key: "value")
+      _id1 = redis.xadd("redstream:stream:products", key: "value")
       id2 = redis.xadd("redstream:stream:products", key: "value")
 
       expect(Redstream.max_stream_id("products")).to eq(id2)
@@ -46,7 +55,7 @@ RSpec.describe Redstream do
     it "returns the consumer's max id" do
       expect(Redstream.max_consumer_id(stream_name: "products", consumer_name: "consumer")).to be_nil
 
-      id1 = redis.xadd("redstream:stream:products", key: "value")
+      _id1 = redis.xadd("redstream:stream:products", key: "value")
       id2 = redis.xadd("redstream:stream:products", key: "value")
 
       Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do |messages|
@@ -114,4 +123,3 @@ RSpec.describe Redstream do
     end
   end
 end
-

data/spec/spec_helper.rb

@@ -1,4 +1,3 @@
-
 require File.expand_path("../lib/redstream", __dir__)
 require "active_record"
 require "factory_bot"
@@ -18,7 +17,7 @@ ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS products"
 
 ActiveRecord::Base.connection.create_table :products do |t|
   t.string :title
-  t.timestamps
+  t.timestamps null: false
 end
 
 class Product < ActiveRecord::Base
@@ -62,5 +61,3 @@ RSpec.configure do |config|
     Redis.new.flushdb
   end
 end
-
-

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: redstream
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-26 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: database_cleaner
+  name: factory_bot
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,21 +81,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.13
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.13
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: factory_bot
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: timecop
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: rspec-instafail
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec-instafail
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -151,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mocha
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -165,13 +165,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: connection_pool
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -193,19 +193,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redis
+  name: connection_pool
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +220,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
 description: Using redis streams to keep your primary database in sync with secondary
   datastores
 email:
@@ -228,8 +242,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
+- ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -272,8 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Using redis streams to keep your primary database in sync with secondary