redstream 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba9468ab2f4b93d53a2f063b3b841137c32ded698a46b0faf1d30aac19baaf1e
-  data.tar.gz: a2cd5dd4067687ad9e3f768b26be016978126037578a8489d6bc8f00d3b55f62
+  metadata.gz: 7c563156ded6459fa0a2c4fcd317d3d5a128a72fe8f0732b358672efec718a3b
+  data.tar.gz: 41c68a6d61e4287d5bffca7bd9277f55ef203d804b609b87de4b85cc7185560c
 SHA512:
-  metadata.gz: b2d36e3a49fde99a438da2648164248f42ed376764073b3b8c2d935854fd8402058a724001a2409aca5cd9e0d0d560aebb381c504aa5fc2148664a989f9b7601
-  data.tar.gz: be66802bd2052f762b4a02c1bfca39f25c4497716fb30a8ba9046df6ba281e59ad75ed6901d90fce9b945dfcdfc8d2dfa2a77b07688cccbd8ed56bfedfcd4297
+  metadata.gz: 812e8cb419ea1a301e462250f20c798f92db02726b46e8eb196e169e8f6dcede85023cbecaa7b9b476d1e18e9b8902bfa68d36b49813dd4dea8f37b202fe9b5b
+  data.tar.gz: e36a3fb9130992efd3cd812ac8b1368ab7dc056cc88016208ec83cc629077650aca935bb2d556396a47cc980f428515713de3bda61fd023867970d55e470755f

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## v0.3.0
+* Pipeline deletion of delay messages
+
 ## v0.2.0
 * Delete delay messages after queue messages are sent
 

data/README.md

@@ -97,7 +97,9 @@ any errors occurring in between `after_save` and `after_commit` result in
 inconsistencies between your primary and secondary datastore. By using these
 kinds of "delay" messages triggered by `after_save` and fetched after e.g. 5
 minutes, errors occurring in between `after_save` and `after_commit` can be
-fixed when the delay message get processed.
+fixed when the delay message get processed. Please note that redstream deletes
+delay messages after the messages for immediate retrieval have been
+successfully sent, such that messages will not be processed twice, usually.
 
 Any messages are fetched in batches, such that e.g. elasticsearch can be
 updated using its bulk API. For instance, depending on which elasticsearch ruby

data/lib/redstream/model.rb

@@ -37,22 +37,14 @@ module Redstream
 
         after_commit(on: [:create, :update]) do |object|
           if object.saved_changes.present?
-            producer.queue(object)
-
-            if id = instance_variable_get(IVAR_DELAY_MESSAGE_ID)
-              producer.delete(object, id)
-              remove_instance_variable(IVAR_DELAY_MESSAGE_ID)
-            end
+            producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
+            instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
           end
         end
 
         after_commit(on: :destroy) do |object|
-          producer.queue(object)
-
-          if id = instance_variable_get(IVAR_DELAY_MESSAGE_ID)
-            producer.delete(object, id)
-            remove_instance_variable(IVAR_DELAY_MESSAGE_ID)
-          end
+          producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
+          instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
         end
       end
 

data/lib/redstream/producer.rb

@@ -52,12 +52,11 @@ module Redstream
     def bulk(records)
       records_array = Array(records)
 
-      message_ids = bulk_delay(records_array)
+      delay_message_ids = bulk_delay(records_array)
 
       yield
 
-      bulk_queue(records_array)
-      bulk_delete(records_array, message_ids)
+      bulk_queue(records_array, delay_message_ids: delay_message_ids)
     end
 
     # @api private
@@ -73,7 +72,7 @@ module Redstream
         Redstream.connection_pool.with do |redis|
           redis.pipelined do
             slice.each do |object|
-              redis.xadd Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload)
+              redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload))
             end
           end
         end
@@ -86,37 +85,19 @@ module Redstream
       res
     end
 
-    # @api private
-    #
-    # Deletes delay message from a delay stream in redis.
-    #
-    # @param records [#to_a] The object/objects that have beeen updated or deleted
-    # @param ids [#to_a] The ids of the respective delay messages
-
-    def bulk_delete(records, ids)
-      records.each_with_index.each_slice(250) do |slice|
-        Redstream.connection_pool.with do |redis|
-          redis.pipelined do
-            slice.each do |object, index|
-              redis.xdel Redstream.stream_key_name("#{stream_name(object)}.delay"), ids[index]
-            end
-          end
-        end
-      end
-    end
-
     # @api private
     #
     # Writes messages to a stream in redis for immediate retrieval.
     #
     # @param records [#to_a] The object/objects that will be updated deleted
 
-    def bulk_queue(records)
-      records.each_slice(250) do |slice|
+    def bulk_queue(records, delay_message_ids:)
+      records.each_with_index.each_slice(250) do |slice|
         Redstream.connection_pool.with do |redis|
           redis.pipelined do
-            slice.each do |object|
-              redis.xadd Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload)
+            slice.each do |object, index|
+              redis.xadd(Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload))
+              redis.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_ids[index]) if delay_message_ids
             end
           end
         end
@@ -141,28 +122,18 @@ module Redstream
       end
     end
 
-    # @api private
-    #
-    # Deletes a single delay message from a delay stream in redis.
-    #
-    # @param object The object that has been updated, deleted, ect.
-    # @param id The redis message id
-
-    def delete(object, id)
-      Redstream.connection_pool.with do |redis|
-        redis.xdel Redstream.stream_key_name("#{stream_name(object)}.delay"), id
-      end
-    end
-
     # @api private
     #
     # Writes a single message to a stream in redis for immediate retrieval.
     #
     # @param object The object hat will be updated, deleted, etc.
 
-    def queue(object)
+    def queue(object, delay_message_id:)
       Redstream.connection_pool.with do |redis|
-        redis.xadd Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload)
+        redis.pipelined do
+          redis.xadd(Redstream.stream_key_name(stream_name(object)), payload: JSON.dump(object.redstream_payload))
+          redis.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_id) if delay_message_id
+        end
       end
 
       true

data/lib/redstream/version.rb

@@ -1,3 +1,3 @@
 module Redstream
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/redstream/producer_spec.rb

@@ -7,9 +7,20 @@ RSpec.describe Redstream::Producer do
 
       stream_key_name = Redstream.stream_key_name("products")
 
-      expect { Redstream::Producer.new.queue(product) }.to change { redis.xlen(stream_key_name) }.by(1)
+      expect { Redstream::Producer.new.queue(product, delay_message_id: nil) }.to change { redis.xlen(stream_key_name) }.by(1)
       expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
     end
+
+    it "deletes the delay message when given" do
+      product = create(:product)
+
+      producer = Redstream::Producer.new
+
+      id = producer.delay(product)
+      producer.queue(product, delay_message_id: id)
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+    end
   end
 
   describe "#delay" do
@@ -31,16 +42,54 @@ RSpec.describe Redstream::Producer do
     end
   end
 
-  describe "#destroy" do
-    it "deletes the delay message for the object" do
-      product = create(:product)
+  describe "#bulk" do
+    it "adds bulk delay messages for scopes" do
+      products = create_list(:product, 2)
+
+      stream_key_name = Redstream.stream_key_name("products")
+
+      expect(redis.xlen("#{stream_key_name}.delay")).to eq(0)
+
+      Redstream::Producer.new.bulk(Product.all) do
+        messages = redis.xrange("#{stream_key_name}.delay", "-", "+").last(2).map { |message| message[1] }
+
+        expect(messages).to eq([
+          { "payload" => JSON.dump(products[0].redstream_payload) },
+          { "payload" => JSON.dump(products[1].redstream_payload) }
+        ])
+      end
+    end
+
+    it "adds bulk queue messages for scopes" do
+      products = create_list(:product, 2)
+
+      stream_key_name = Redstream.stream_key_name("products")
 
+      expect do
+        Redstream::Producer.new.bulk(Product.all) do
+          # nothing
+        end
+      end.to change { redis.xlen(stream_key_name) }.by(2)
+
+      messages = redis.xrange(stream_key_name, "-", "+").last(2).map { |message| message[1] }
+
+      expect(messages).to eq([
+        { "payload" => JSON.dump(products[0].redstream_payload) },
+        { "payload" => JSON.dump(products[1].redstream_payload) }
+      ])
+    end
+
+    it "deletes the delay messages after the queue messages have been sent" do
+      products = create_list(:product, 2)
       producer = Redstream::Producer.new
 
-      id = producer.delay(product)
-      producer.delete(product, id)
+      other_delay_message_id = producer.delay(create(:product))
 
-      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+      producer.bulk(products) do
+        expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(3)
+      end
+
+      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
     end
   end
 
@@ -50,7 +99,7 @@ RSpec.describe Redstream::Producer do
 
       stream_key_name = Redstream.stream_key_name("products")
 
-      expect { Redstream::Producer.new.bulk_queue(Product.all) }.to change { redis.xlen(stream_key_name) }.by(2)
+      expect { Redstream::Producer.new.bulk_queue(Product.all, delay_message_ids: nil) }.to change { redis.xlen(stream_key_name) }.by(2)
 
       messages = redis.xrange(stream_key_name, "-", "+").last(2).map { |message| message[1] }
 
@@ -64,13 +113,12 @@ RSpec.describe Redstream::Producer do
       products = create_list(:product, 2)
       producer = Redstream::Producer.new
 
-      other_id = producer.delay(create(:product))
+      delay_message_ids = producer.bulk_delay(products)
+      other_delay_message_id = producer.delay(create(:product))
 
-      producer.bulk_queue(products) do
-        expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(2)
-      end
+      producer.bulk_queue(products, delay_message_ids: delay_message_ids)
 
-      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_id])
+      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
     end
   end
 
@@ -100,18 +148,4 @@ RSpec.describe Redstream::Producer do
       expect { Redstream::Producer.new(wait: 0).bulk_delay(products) }.to change { redis.xlen(stream_key_name) }.by(2)
     end
   end
-
-  describe "#bulk_delete" do
-    it "deletes delay messages for scopes" do
-      products = create_list(:product, 2)
-      producer = Redstream::Producer.new
-
-      other_id = producer.delay(create(:product))
-
-      ids = producer.bulk_delay(products)
-      producer.bulk_delete(products, ids)
-
-      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_id])
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redstream
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord