redstream 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6be694edcad63b15c2e7a6630047080556a412fc1b52f874e4f793e8f07f80b9
-  data.tar.gz: f140446d6fbebc97533de509ff622fe32b121cd89af2c5f0cc2b856f5e820e02
+  metadata.gz: ba9468ab2f4b93d53a2f063b3b841137c32ded698a46b0faf1d30aac19baaf1e
+  data.tar.gz: a2cd5dd4067687ad9e3f768b26be016978126037578a8489d6bc8f00d3b55f62
 SHA512:
-  metadata.gz: 2b38daa5a07f1761fa47340c4d479b974b30937479a6a73897ad8d8e47c9d45328ef976d20b8a74a7cb0488d35985864d162b3adbb66133e56195cb016b09ab4
-  data.tar.gz: 0b625e0f6fea967e69ba319115dd9bfd727b68aec369a83a0842bb39805b0389ebef33bae5cf2559314d54724a9fd28d5a517ab8cbdc88f14db1ec5a601c12ee
+  metadata.gz: b2d36e3a49fde99a438da2648164248f42ed376764073b3b8c2d935854fd8402058a724001a2409aca5cd9e0d0d560aebb381c504aa5fc2148664a989f9b7601
+  data.tar.gz: be66802bd2052f762b4a02c1bfca39f25c4497716fb30a8ba9046df6ba281e59ad75ed6901d90fce9b945dfcdfc8d2dfa2a77b07688cccbd8ed56bfedfcd4297

data/.github/workflows/test.yml

@@ -0,0 +1,28 @@
+name: test
+on: [push, pull_request]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ['2.5', '2.6', '2.7']
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - uses: actions/cache@v1
+      id: cache
+      with:
+        path: vendor/bundler
+        key: ${{ hashFiles('Gemfile.lock') }}-${{ matrix.ruby }}
+    - run: |
+        gem install bundler
+        bundle install --path=vendor/bundler
+        bundle exec rspec
+        bundle exec rubocop

data/.rubocop.yml

@@ -1,3 +1,12 @@
+AllCops:
+  NewCops: enable
+
+Lint/AssignmentInCondition:
+  Enabled: false
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
 Lint/AmbiguousBlockAssociation:
   Enabled: false
 
@@ -13,9 +22,6 @@ Layout/ArgumentAlignment:
 Layout/FirstArrayElementIndentation:
   EnforcedStyle: consistent
 
-Style/BracesAroundHashParameters:
-  EnforcedStyle: no_braces
-
 Style/PercentLiteralDelimiters:
   Enabled: false
 
@@ -52,7 +58,7 @@ Style/SymbolArray:
 Layout/RescueEnsureAlignment:
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Enabled: false
 
 Metrics/MethodLength:

data/.travis.yml

@@ -1,7 +1,8 @@
 sudo: false
 language: ruby
+cache: bundler
 rvm:
-  - ruby-head
+  - ruby-2.6.2
 before_install:
   - docker-compose up -d
   - sleep 10

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## v0.2.0
+* Delete delay messages after queue messages are sent
+
 ## v0.1.1
 * Fix missing queue message in `after_commit on: :destroy`
 

data/README.md

@@ -4,7 +4,7 @@
 **Using redis streams to keep your primary database in sync with secondary
 datastores (e.g. elasticsearch).**
 
-[![Build Status](https://secure.travis-ci.org/mrkamel/redstream.png?branch=master)](http://travis-ci.org/mrkamel/redstream)
+[![Build Status](https://github.com/mrkamel/redstream/workflows/test/badge.svg?branch=master)](https://github.com/mrkamel/redstream/actions?query=workflow%3Atest)
 
 ## Installation
 
@@ -86,9 +86,9 @@ end
 More concretely, `after_save`, `after_touch` and `after_destroy` only write
 "delay" messages to an additional redis stream. Delay message are like any
 other messages, but they get processed by a `Redstream::Delayer` and the
-`Delayer`will wait for some (configurable) delay/time before processing them.
+`Delayer` will wait for some (configurable) delay/time before processing them.
 As the `Delayer` is neccessary to fix inconsistencies, the delay must be at
-least as long as your maxiumum database transaction time. Contrary,
+least as long as your maximum database transaction time. Contrary,
 `after_commit` writes messages to a redis stream from which the messages can
 be fetched immediately to keep the secondary datastores updated in
 near-realtime. The reasoning of all this is simple: usually, i.e. by using only

data/lib/redstream/model.rb

@@ -12,6 +12,8 @@ module Redstream
   #   end
 
   module Model
+    IVAR_DELAY_MESSAGE_ID = :@__redstream_delay_message_id__
+
     def self.included(base)
       base.extend(ClassMethods)
     end
@@ -29,11 +31,29 @@ module Redstream
       #   responsible for writing to a redis stream
 
       def redstream_callbacks(producer: Producer.new)
-        after_save { |object| producer.delay(object) if object.saved_changes.present? }
-        after_touch { |object| producer.delay(object) }
-        after_destroy { |object| producer.delay(object) }
-        after_commit(on: [:create, :update]) { |object| producer.queue(object) if object.saved_changes.present? }
-        after_commit(on: :destroy) { |object| producer.queue(object) }
+        after_save    { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) if object.saved_changes.present? }
+        after_touch   { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
+        after_destroy { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
+
+        after_commit(on: [:create, :update]) do |object|
+          if object.saved_changes.present?
+            producer.queue(object)
+
+            if id = instance_variable_get(IVAR_DELAY_MESSAGE_ID)
+              producer.delete(object, id)
+              remove_instance_variable(IVAR_DELAY_MESSAGE_ID)
+            end
+          end
+        end
+
+        after_commit(on: :destroy) do |object|
+          producer.queue(object)
+
+          if id = instance_variable_get(IVAR_DELAY_MESSAGE_ID)
+            producer.delete(object, id)
+            remove_instance_variable(IVAR_DELAY_MESSAGE_ID)
+          end
+        end
       end
 
       def redstream_name

data/lib/redstream/producer.rb

@@ -52,11 +52,12 @@ module Redstream
     def bulk(records)
       records_array = Array(records)
 
-      bulk_delay(records_array)
+      message_ids = bulk_delay(records_array)
 
       yield
 
       bulk_queue(records_array)
+      bulk_delete(records_array, message_ids)
     end
 
     # @api private
@@ -64,12 +65,14 @@ module Redstream
     # Writes delay messages to a delay stream in redis.
     #
     # @param records [#to_a] The object/objects that will be updated or deleted
+    #
+    # @return The redis message ids
 
     def bulk_delay(records)
-      records.each_slice(250) do |slice|
+      res = records.each_slice(250).flat_map do |slice|
         Redstream.connection_pool.with do |redis|
           redis.pipelined do
-            slice.map do |object|
+            slice.each do |object|
               redis.xadd Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload)
             end
           end
@@ -80,7 +83,26 @@ module Redstream
         redis.wait(@wait, 0) if @wait
       end
 
-      true
+      res
+    end
+
+    # @api private
+    #
+    # Deletes delay message from a delay stream in redis.
+    #
+    # @param records [#to_a] The object/objects that have beeen updated or deleted
+    # @param ids [#to_a] The ids of the respective delay messages
+
+    def bulk_delete(records, ids)
+      records.each_with_index.each_slice(250) do |slice|
+        Redstream.connection_pool.with do |redis|
+          redis.pipelined do
+            slice.each do |object, index|
+              redis.xdel Redstream.stream_key_name("#{stream_name(object)}.delay"), ids[index]
+            end
+          end
+        end
+      end
     end
 
     # @api private
@@ -107,15 +129,29 @@ module Redstream
     #
     # Writes a single delay message to a delay stream in redis.
     #
-    # @param object The object hat will be updated, deleted, etc.
+    # @param object The object that will be updated, deleted, etc.
+    #
+    # @return The redis message id
 
     def delay(object)
       Redstream.connection_pool.with do |redis|
-        redis.xadd Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload)
+        res = redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload))
         redis.wait(@wait, 0) if @wait
+        res
       end
+    end
 
-      true
+    # @api private
+    #
+    # Deletes a single delay message from a delay stream in redis.
+    #
+    # @param object The object that has been updated, deleted, ect.
+    # @param id The redis message id
+
+    def delete(object, id)
+      Redstream.connection_pool.with do |redis|
+        redis.xdel Redstream.stream_key_name("#{stream_name(object)}.delay"), id
+      end
     end
 
     # @api private

data/lib/redstream/version.rb

@@ -1,3 +1,3 @@
 module Redstream
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/spec/redstream/consumer_spec.rb

@@ -80,7 +80,9 @@ RSpec.describe Redstream::Consumer do
 
       all_messages = redis.xrange(Redstream.stream_key_name("products"), "-", "+")
 
-      Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once {}
+      Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do
+        # nothing
+      end
 
       expect(redis.get(Redstream.offset_key_name(stream_name: "products", consumer_name: "consumer"))).to eq(all_messages.last[0])
     end

data/spec/redstream/model_spec.rb

@@ -3,19 +3,36 @@ require File.expand_path("../spec_helper", __dir__)
 RSpec.describe Redstream::Model do
   describe "after_save" do
     it "adds a delay message after_save" do
-      expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      Product.transaction do
+        expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      end
+    end
+
+    it "assigns the delay message id" do
+      Product.transaction do
+        expect(create(:product).instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
+      end
     end
 
     it "adds the correct payload for the delay message" do
-      product = create(:product)
+      Product.transaction do
+        product = create(:product)
 
-      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+        expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      end
     end
 
     it "adds a queue message after_save on commit" do
       expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products")) }
     end
 
+    it "deletes the delay message on commit" do
+      product = create(:product)
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
+    end
+
     it "does not add a delay message after_save if there are no changes" do
       product = create(:product)
 
@@ -33,35 +50,73 @@ RSpec.describe Redstream::Model do
     it "adds a delay message after touch" do
       product = create(:product)
 
-      expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      Product.transaction do
+        expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      end
+    end
+
+    it "assigns the delay message id" do
+      product = create(:product)
+
+      Product.transaction do
+        product.touch
+
+        expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
+      end
     end
 
     it "sets the correct payload for the delay message" do
       product = create(:product)
-      product.touch
 
-      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      Product.transaction do
+        product.touch
+
+        expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      end
     end
 
-    it "adds a queue message after touch commit" do
+    it "adds a queue message after touch on commit" do
       product = create(:product)
 
       expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products")) }
     end
+
+    it "deletes the delay message after touch on commit" do
+      product = create(:product)
+      product.touch
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
+    end
   end
 
   describe "after_destroy" do
     it "adds a delay message after destroy" do
       product = create(:product)
 
-      expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      Product.transaction do
+        expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
+      end
+    end
+
+    it "assigns the delay message id" do
+      product = create(:product)
+
+      Product.transaction do
+        product.destroy
+
+        expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
+      end
     end
 
     it "sets the correct payload for the delay message" do
       product = create(:product)
-      product.destroy
 
-      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      Product.transaction do
+        product.destroy
+
+        expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
+      end
     end
 
     it "adds a queue messages after destroy on commit" do
@@ -69,5 +124,13 @@ RSpec.describe Redstream::Model do
 
       expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products")) }.by(1)
     end
+
+    it "deletes the delay message after destroy on commit" do
+      product = create(:product)
+      product.destroy
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+      expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
+    end
   end
 end

data/spec/redstream/producer_spec.rb

@@ -31,6 +31,19 @@ RSpec.describe Redstream::Producer do
     end
   end
 
+  describe "#destroy" do
+    it "deletes the delay message for the object" do
+      product = create(:product)
+
+      producer = Redstream::Producer.new
+
+      id = producer.delay(product)
+      producer.delete(product, id)
+
+      expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
+    end
+  end
+
   describe "#bulk_queue" do
     it "adds bulk queue messages for scopes" do
       products = create_list(:product, 2)
@@ -46,6 +59,19 @@ RSpec.describe Redstream::Producer do
         { "payload" => JSON.dump(products[1].redstream_payload) }
       ])
     end
+
+    it "deletes the delay messages after the queue messages have been sent" do
+      products = create_list(:product, 2)
+      producer = Redstream::Producer.new
+
+      other_id = producer.delay(create(:product))
+
+      producer.bulk_queue(products) do
+        expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(2)
+      end
+
+      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_id])
+    end
   end
 
   describe "#bulk_delay" do
@@ -64,7 +90,7 @@ RSpec.describe Redstream::Producer do
       ])
     end
 
-    it "should resepect wait for delay" do
+    it "should respect wait for delay" do
       create(:product)
 
       stream_key_name = Redstream.stream_key_name("products.delay")
@@ -74,4 +100,18 @@ RSpec.describe Redstream::Producer do
       expect { Redstream::Producer.new(wait: 0).bulk_delay(products) }.to change { redis.xlen(stream_key_name) }.by(2)
     end
   end
+
+  describe "#bulk_delete" do
+    it "deletes delay messages for scopes" do
+      products = create_list(:product, 2)
+      producer = Redstream::Producer.new
+
+      other_id = producer.delay(create(:product))
+
+      ids = producer.bulk_delay(products)
+      producer.bulk_delete(products, ids)
+
+      expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_id])
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -17,7 +17,7 @@ ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS products"
 
 ActiveRecord::Base.connection.create_table :products do |t|
   t.string :title
-  t.timestamps
+  t.timestamps null: false
 end
 
 class Product < ActiveRecord::Base

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redstream
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Benjamin Vetter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-20 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -242,6 +242,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"