rapidity 0.0.5.88265 → 0.0.6.88627

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6faa9172962dc6c7004e59dc8e827f16572b99c41b586c7221539d2bf253dda5
-  data.tar.gz: fb698e043084b6ad4491c2c85b785f9678aba9996f333e3df6bae45d6f76a76a
+  metadata.gz: 4812c1514ceb875e24c9c4a9b86ca1eaef1227eceb13f8b1ae7498c7579e6809
+  data.tar.gz: 0b812cebd0ed83cdfa35e52550d73b42fbabed89b60e771695c02cf0720d71e6
 SHA512:
-  metadata.gz: d76e69e666ae0ab3a541f8ce725b2aae485ac9c7218163bd23bcaf4023f33db748da42f5dab7b978699f3f0ac3c0cfce9b49e2933127b81613d7df4533bc7496
-  data.tar.gz: 7434ad4f51dc20142d28f5df0bd004f79e8674dec8ac317683d18921626469525f81956ce4c3b7a8260c11cdb3cd7f281bb3f7f7aac5123e532899a9446d1587
+  metadata.gz: b037c65bed50ed704dace8fcc16cfa554e39c810d72b9f1da7b430b2a1bad6f1d7799bfd2e0f8615ece8f45b6fa2457445e94f2996e3a1caac31d0f5b9ecac52
+  data.tar.gz: cb0ada7d5095aac7bd0c447f95d7de55628b6574cc7b07535ff548d64d3cf9dc640fa8d780772892dc59d4e90ad44d54cee1f4808799bda447929ef874677c1f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rapidity (0.0.5.88265)
+    rapidity (0.0.6)
       activesupport
       connection_pool
       redis
@@ -9,13 +9,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3.2)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ansi (1.5.0)
     ast (2.4.2)
@@ -28,62 +27,60 @@ GEM
     byebug (11.1.3)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.1.10)
     connection_pool (2.2.5)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     docile (1.4.0)
     equalizer (0.0.11)
-    erubis (2.7.0)
-    flay (2.12.1)
-      erubis (~> 2.7.0)
+    erubi (1.10.0)
+    flay (2.13.0)
+      erubi (~> 1.10)
       path_expander (~> 1.0)
       ruby_parser (~> 3.0)
       sexp_processor (~> 4.0)
-    flog (4.6.4)
+    flog (4.6.5)
       path_expander (~> 1.0)
       ruby_parser (~> 3.1, > 3.1.0)
       sexp_processor (~> 4.8)
-    i18n (1.8.10)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     kwalify (0.7.2)
     launchy (2.5.0)
       addressable (~> 2.7)
-    minitest (5.14.4)
-    parser (3.0.1.1)
+    minitest (5.16.1)
+    parser (3.1.2.0)
       ast (~> 2.4.1)
     path_expander (1.1.0)
-    psych (3.3.2)
-    public_suffix (4.0.6)
-    rainbow (3.0.0)
+    public_suffix (4.0.7)
+    rainbow (3.1.1)
     redis (4.7.0)
-    reek (6.0.4)
+    reek (6.1.1)
       kwalify (~> 0.7.0)
-      parser (~> 3.0.0)
-      psych (~> 3.1)
+      parser (~> 3.1.0)
       rainbow (>= 2.0, < 4.0)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
     rspec-collection_matchers (1.2.0)
       rspec-expectations (>= 2.99.0.beta1)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rspec_junit_formatter (0.4.1)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rspec_junit_formatter (0.5.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    ruby_parser (3.16.0)
-      sexp_processor (~> 4.15, >= 4.15.1)
-    rubycritic (4.6.1)
+    ruby_parser (3.19.1)
+      sexp_processor (~> 4.16)
+    rubycritic (4.7.0)
       flay (~> 2.8)
       flog (~> 4.4)
       launchy (>= 2.0.0)
@@ -94,9 +91,9 @@ GEM
       simplecov (>= 0.17.0)
       tty-which (~> 0.4.0)
       virtus (~> 1.0)
-    sexp_processor (4.15.3)
-    shoulda-matchers (4.5.1)
-      activesupport (>= 4.2.0)
+    sexp_processor (4.16.1)
+    shoulda-matchers (5.1.0)
+      activesupport (>= 5.2.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -106,21 +103,20 @@ GEM
       simplecov
       terminal-table
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    terminal-table (3.0.1)
+    simplecov_json_formatter (0.1.4)
+    terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     thread_safe (0.3.6)
     timeouter (0.1.3.38794)
     tty-which (0.4.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.0.0)
+    unicode-display_width (2.2.0)
     virtus (1.0.5)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
       equalizer (~> 0.0, >= 0.0.9)
-    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby

data/README.md

@@ -12,10 +12,12 @@
 Simple but fast Redis-backed distributed rate limiter. Allows you to specify time interval and count within to limit distributed operations.
 
 Features:
- 
- - extremly simple
- - safe
- - fast
+
+- extremly simple
+- free from race condition through LUA scripting
+- fast
+
+[Article(russian) about gem.](https://blog.rnds.pro/029-rapidity/?utm_source=github&utm_medium=repo&utm_campaign=rnds)
 
 ## Usage
 
@@ -75,18 +77,22 @@ loop do
     sleep 1
   end
 end
-
 ```
 
 ## Installation
 
 It's a gem:
+
 ```bash
   gem install rapidity
 ```
+
 There's also the wonders of [the Gemfile](http://bundler.io):
+
 ```ruby
   gem 'rapidity'
 ```
 
+## Special Thanks
 
+- [WeTransfer/prorate](https://github.com/WeTransfer/prorate) for LUA-examples

data/lib/rapidity/composer.rb

@@ -41,13 +41,18 @@ module Rapidity
       end
     end
 
-    def obtain(count = 5)
+    def obtain(count = 5, with_time: false)
+      time = nil
       @limiters.each do |limiter|
-        count = limiter.obtain(count)
+        count, time = limiter.obtain(count, with_time: with_time)
         break if count == 0
       end
 
-      count
+      if with_time
+        [count, time]
+      else
+        count
+      end
     end
 
   end

data/lib/rapidity/limiter.lua

@@ -0,0 +1,41 @@
+-- args: key, treshold, interval, count
+-- returns: obtained count.
+
+-- this is required to be able to use TIME and writes; basically it lifts the script into IO
+redis.replicate_commands()
+
+-- make some nicer looking variable names:
+local retval = nil
+
+-- Redis documentation recommends passing the keys separately so that Redis
+-- can - in the future - verify that they live on the same shard of a cluster, and
+-- raise an error if they are not. As far as can be understood this functionality is not
+-- yet present, but if we can make a little effort to make ourselves more future proof
+-- we should.
+local key = KEYS[1]
+local treshold = tonumber(ARGV[1])
+local interval = tonumber(ARGV[2])
+local count = tonumber(ARGV[3])
+
+local current = 0
+local to_return = 0
+
+local redis_time = redis.call("TIME") -- Array of [seconds, microseconds]
+redis.call("SET", key, treshold, "EX", interval, "NX")
+current = redis.call("DECRBY", key, count)
+
+-- If we became below zero we must return some value back
+if current < 0 then
+  to_return = math.min(count, math.abs(current))
+
+  -- set 0 to current counter value
+  redis.call("SET", key, 0, 'KEEPTTL')
+
+  -- return obtained part of requested count
+  retval = count - to_return
+else
+  -- return full of requested count
+  retval = count
+end
+
+return {retval, redis_time}

data/lib/rapidity/limiter.rb

@@ -6,6 +6,7 @@ module Rapidity
 
     attr_reader :pool, :name, :interval, :threshold, :namespace
 
+    LUA_SCRIPT_CODE = File.read(File.join(__dir__, 'limiter.lua'))
 
     # Convert message to given class
     # @params pool - inititalized Redis pool
@@ -30,51 +31,61 @@ module Rapidity
     # @return remaining counter value
     def remains
       results = @pool.with do |conn|
-        conn.multi do
-          conn.set(key('remains'), threshold, ex: interval, nx: true)
-          conn.get(key('remains'))
+        conn.multi do |pipeline|
+          pipeline.set(key('remains'), threshold, ex: interval, nx: true)
+          pipeline.get(key('remains'))
         end
       end
-      results[1].to_i #=> conn.get(key('remains'))
+      results[1].to_i #=> pipeline.get(key('remains'))
     end
 
     # Obtain values from counter
+    # @params count - try to obtain 'tokens'
+    # @params with_time - return or not Redis-service time at which request processed
     # @return count succesfuly obtained send slots
-    def obtain(count = 5)
+    def obtain(count = 5, with_time: false)
       count = count.abs
 
-      results = @pool.with do |conn|
-        conn.multi do
-          conn.set(key('remains'), threshold, ex: interval, nx: true)
-          conn.decrby(key('remains'), count)
+      result = begin
+        @pool.with do |conn|
+          conn.evalsha(@script, keys: [key('remains')], argv: [threshold, interval, count])
+        end
+      rescue Redis::CommandError => e
+        if e.message.include?('NOSCRIPT')
+          # The Redis server has never seen this script before. Needs to run only once in the entire lifetime
+          # of the Redis server, until the script changes - in which case it will be loaded under a different SHA
+          ensure_script_loaded
+          retry
+        else
+          raise e
         end
       end
 
-      taken = results[1].to_i #=> conn.decrby(key('remains'), count)
+      taken = result[0].to_i
+      time = Time.at(*result[1].map(&:to_i), :millisecond)
 
-      if taken < 0
-        overflow = taken.abs
-        to_return = [count, overflow].min
-
-        results = @pool.with do |conn|
-          conn.multi do
-            conn.set(key('remains'), threshold - to_return, ex: interval, nx: true)
-            conn.incrby(key('remains'), to_return)
-            conn.ttl(key('remains'))
-          end
+      if taken == 0
+        ttl = @pool.with do |conn|
+          conn.ttl(key('remains'))
         end
 
-        ttl = results[2].to_i #=> conn.ttl(key('remains'))
-
-        # reset if no ttl present
-        if ttl == -1 
+        # UNKNOWN BUG? reset if no ttl present. Many years ago once upon time we meet our key without TTL
+        if ttl == -1
           STDERR.puts "ERROR[#{Time.now}]: TTL for key #{key('remains').inspect} disappeared!"
-          @pool.with {|c| c.expire(key('remains'), interval) } 
+          @pool.with {|c| c.expire(key('remains'), interval) }
         end
+      end
 
-        count - to_return
+      if with_time
+        [taken, time]
       else
-        count
+        taken
+      end
+    end
+
+    def ensure_script_loaded
+      @script = @pool.with do |conn|
+        conn.script(:load, LUA_SCRIPT_CODE)
       end
     end
 

data/lib/rapidity/version.rb

@@ -1,6 +1,6 @@
 module Rapidity
 
-  VERSION = '0.0.5'.freeze
+  VERSION = '0.0.6'.freeze
 
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rapidity
 version: !ruby/object:Gem::Version
-  version: 0.0.5.88265
+  version: 0.0.6.88627
 platform: ruby
 authors:
 - Yurusov Vlad
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-25 00:00:00.000000000 Z
+date: 2022-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -235,6 +235,7 @@ files:
 - README.md
 - lib/rapidity.rb
 - lib/rapidity/composer.rb
+- lib/rapidity/limiter.lua
 - lib/rapidity/limiter.rb
 - lib/rapidity/version.rb
 homepage: