redstream 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 032fbb3b9ee72cf051d9b9dffe996d1efee73b106952d7739bfe75a08190ac2c
-  data.tar.gz: da3131c6117fb981d1709d5825b94bc09f59cd771835cfd0516cfec22765d09a
+  metadata.gz: '08b45ec057fc3fa2404ff0c48c9f1befdd7dee5177b526690cb5edb907990ac7'
+  data.tar.gz: '081b1cbd5ed872b3d320b1ffe6e7f4342c5bfcb101b293a5cf9348241f8dfe44'
 SHA512:
-  metadata.gz: 55bf7e981765709911f69af1c30a8ee00dc1138bc0b6636ef6ec3442c5428138eebfb23553069cae3be520e1d8b5c40594db9c3c6646d18555834761813e4551
-  data.tar.gz: 4dda09649b7f523f57a19dc8e17a8c3c9a9177199c26b8530db45fa634d1bc0e589ddc44eeddb4fc8f67a81a3e03d81f09ac6d432b1bb60d39e943bb27a22f9e
+  metadata.gz: df13ccbbfe9b62e79d5dab0ac461f9c433aff8b6a2990079a4f5c007cb72e93cecf95fd65eb5aa9780da3542a95be01aa7ab83ac109b19b3af93faed69678e58
+  data.tar.gz: ab2e5e8cc1c2ec8b8031cd27621a53e302b1ca23b1d50accb17a348742a0cebdefbf9d92adc6f216534dfcd145262f098f128e3539fd64d57fae994e54745f5e

data/.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
           - 6379:6379
     steps:
     - uses: actions/checkout@v1
-    - uses: actions/setup-ruby@v1
+    - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
     - uses: actions/cache@v1

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## v0.5.0
+* No longer delete delay messages
+
+## v0.4.4
+* Remove deletion of delay messages after queues messages are sent from `.bulk`
+
 ## v0.4.3
 * Change gemspec to support older redis versions again
 

data/README.md

@@ -97,9 +97,7 @@ any errors occurring in between `after_save` and `after_commit` result in
 inconsistencies between your primary and secondary datastore. By using these
 kinds of "delay" messages triggered by `after_save` and fetched after e.g. 5
 minutes, errors occurring in between `after_save` and `after_commit` can be
-fixed when the delay message get processed. Please note that redstream deletes
-delay messages after the messages for immediate retrieval have been
-successfully sent, such that messages will not be processed twice, usually.
+fixed when the delay message get processed.
 
 Any messages are fetched in batches, such that e.g. elasticsearch can be
 updated using its bulk API. For instance, depending on which elasticsearch ruby

data/lib/redstream/model.rb

@@ -12,8 +12,6 @@ module Redstream
   #   end
 
   module Model
-    IVAR_DELAY_MESSAGE_ID = :@__redstream_delay_message_id__
-
     def self.included(base)
       base.extend(ClassMethods)
     end
@@ -31,20 +29,16 @@ module Redstream
       #   responsible for writing to a redis stream
 
       def redstream_callbacks(producer: Producer.new)
-        after_save    { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) if object.saved_changes.present? }
-        after_touch   { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
-        after_destroy { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
+        after_save    { |object| producer.delay(object) if object.saved_changes.present? }
+        after_touch   { |object| producer.delay(object) }
+        after_destroy { |object| producer.delay(object) }
 
         after_commit(on: [:create, :update]) do |object|
-          if object.saved_changes.present?
-            producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
-            instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
-          end
+          producer.queue(object) if object.saved_changes.present?
         end
 
         after_commit(on: :destroy) do |object|
-          producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
-          instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
+          producer.queue(object)
         end
       end
 

data/lib/redstream/producer.rb

@@ -52,11 +52,11 @@ module Redstream
     def bulk(records)
       records_array = Array(records)
 
-      delay_message_ids = bulk_delay(records_array)
+      bulk_delay(records_array)
 
       yield
 
-      bulk_queue(records_array, delay_message_ids: delay_message_ids)
+      bulk_queue(records_array)
     end
 
     # @api private
@@ -68,7 +68,7 @@ module Redstream
     # @return The redis message ids
 
     def bulk_delay(records)
-      res = records.each_slice(250).flat_map do |slice|
+      records.each_slice(250) do |slice|
         Redstream.connection_pool.with do |redis|
           redis.pipelined do |pipeline|
             slice.each do |object|
@@ -82,7 +82,7 @@ module Redstream
         redis.wait(@wait, 0) if @wait
       end
 
-      res
+      true
     end
 
     # @api private
@@ -90,15 +90,13 @@ module Redstream
     # Writes messages to a stream in redis for immediate retrieval.
     #
     # @param records [#to_a] The object/objects that will be updated deleted
-    # @param delay_message_ids [#to_a] The delay message ids to delete
 
-    def bulk_queue(records, delay_message_ids: nil)
-      records.each_with_index.each_slice(250) do |slice|
+    def bulk_queue(records)
+      records.each_slice(250) do |slice|
         Redstream.connection_pool.with do |redis|
           redis.pipelined do |pipeline|
-            slice.each do |object, index|
+            slice.each do |object|
               pipeline.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
-              pipeline.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_ids[index]) if delay_message_ids
             end
           end
         end
@@ -128,14 +126,10 @@ module Redstream
     # Writes a single message to a stream in redis for immediate retrieval.
     #
     # @param object The object hat will be updated, deleted, etc.
-    # @param delay_message_id The delay message id to delete
 
-    def queue(object, delay_message_id: nil)
+    def queue(object)
       Redstream.connection_pool.with do |redis|
-        redis.pipelined do |pipeline|
-          pipeline.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
-          pipeline.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_id) if delay_message_id
-        end
+        redis.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
       end
 
       true

data/lib/redstream/version.rb

@@ -1,3 +1,3 @@
 module Redstream
-  VERSION = "0.4.3"
+  VERSION = "0.5.0"
 end

data/redstream.gemspec

@@ -14,7 +14,6 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "activerecord"

data/spec/redstream/model_spec.rb

@@ -8,12 +8,6 @@ RSpec.describe Redstream::Model do
       end
     end
 
-    it "assigns the delay message id" do
-      Product.transaction do
-        expect(create(:product).instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
-      end
-    end
-
     it "adds the correct payload for the delay message" do
       Product.transaction do
         product = create(:product)
@@ -26,13 +20,6 @@ RSpec.describe Redstream::Model do
       expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products")) }
     end
 
-    it "deletes the delay message on commit" do
-      product = create(:product)
-
-      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
-      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
-    end
-
     it "does not add a delay message after_save if there are no changes" do
       product = create(:product)
 
@@ -55,16 +42,6 @@ RSpec.describe Redstream::Model do
       end
     end
 
-    it "assigns the delay message id" do
-      product = create(:product)
-
-      Product.transaction do
-        product.touch
-
-        expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
-      end
-    end
-
     it "sets the correct payload for the delay message" do
       product = create(:product)
 
@@ -80,14 +57,6 @@ RSpec.describe Redstream::Model do
 
       expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products")) }
     end
-
-    it "deletes the delay message after touch on commit" do
-      product = create(:product)
-      product.touch
-
-      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
-      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
-    end
   end
 
   describe "after_destroy" do
@@ -99,16 +68,6 @@ RSpec.describe Redstream::Model do
       end
     end
 
-    it "assigns the delay message id" do
-      product = create(:product)
-
-      Product.transaction do
-        product.destroy
-
-        expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
-      end
-    end
-
     it "sets the correct payload for the delay message" do
       product = create(:product)
 
@@ -124,13 +83,5 @@ RSpec.describe Redstream::Model do
 
       expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products")) }.by(1)
     end
-
-    it "deletes the delay message after destroy on commit" do
-      product = create(:product)
-      product.destroy
-
-      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
-      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
-    end
   end
 end

data/spec/redstream/producer_spec.rb

@@ -10,17 +10,6 @@ RSpec.describe Redstream::Producer do
       expect { Redstream::Producer.new.queue(product) }.to change { redis.xlen(stream_key_name) }.by(1)
       expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
     end
-
-    it "deletes the delay message when given" do
-      product = create(:product)
-
-      producer = Redstream::Producer.new
-
-      id = producer.delay(product)
-      producer.queue(product, delay_message_id: id)
-
-      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
-    end
   end
 
   describe "#delay" do
@@ -48,9 +37,11 @@ RSpec.describe Redstream::Producer do
 
       stream_key_name = Redstream.stream_key_name("products")
 
-      expect(redis.xlen("#{stream_key_name}.delay")).to eq(0)
+      expect(redis.xlen("#{stream_key_name}.delay")).to eq(2)
 
       Redstream::Producer.new.bulk(Product.all) do
+        expect(redis.xlen("#{stream_key_name}.delay")).to eq(4)
+
         messages = redis.xrange("#{stream_key_name}.delay", "-", "+").last(2).map { |message| message[1] }
 
         expect(messages).to eq([
@@ -78,19 +69,6 @@ RSpec.describe Redstream::Producer do
         { "payload" => JSON.dump(products[1].redstream_payload) }
       ])
     end
-
-    it "deletes the delay messages after the queue messages have been sent" do
-      products = create_list(:product, 2)
-      producer = Redstream::Producer.new
-
-      other_delay_message_id = producer.delay(create(:product))
-
-      producer.bulk(products) do
-        expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(3)
-      end
-
-      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
-    end
   end
 
   describe "#bulk_queue" do
@@ -108,18 +86,6 @@ RSpec.describe Redstream::Producer do
         { "payload" => JSON.dump(products[1].redstream_payload) }
       ])
     end
-
-    it "deletes the delay messages after the queue messages have been sent" do
-      products = create_list(:product, 2)
-      producer = Redstream::Producer.new
-
-      delay_message_ids = producer.bulk_delay(products)
-      other_delay_message_id = producer.delay(create(:product))
-
-      producer.bulk_queue(products, delay_message_ids: delay_message_ids)
-
-      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
-    end
   end
 
   describe "#bulk_delay" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redstream
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-28 00:00:00.000000000 Z
+date: 2022-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -294,12 +294,4 @@ signing_key:
 specification_version: 4
 summary: Using redis streams to keep your primary database in sync with secondary
   datastores
-test_files:
-- spec/redstream/consumer_spec.rb
-- spec/redstream/delayer_spec.rb
-- spec/redstream/lock_spec.rb
-- spec/redstream/model_spec.rb
-- spec/redstream/producer_spec.rb
-- spec/redstream/trimmer_spec.rb
-- spec/redstream_spec.rb
-- spec/spec_helper.rb
+test_files: []