redis-ick 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8a5daa6d8971a38cd1612550866901e98cab6a9aa217dbda97e4072cb5f372f
-  data.tar.gz: 9d3909a5c10d7729ae29ccbc1c55090bfa701885e7156b563c1289e6e51ca427
+  metadata.gz: 5a3b91d98498be4ca9b767fa2af1a90aeccb041038d50e1df47eb8d0218ab4d5
+  data.tar.gz: 439ccf38a34252f6b1dd52c8d44cad3bb2b3f4cdbb9955455db966b51cc73cc4
 SHA512:
-  metadata.gz: 12cd3ee7acca87ee3d09c3569ef214b5073dd768f2fb10a8650435573f0f5d8c27fa02b628bec772fd42c1aad4f70632ddaa7ae554e35be09744e1cd89dc4539
-  data.tar.gz: 579ce38a15bc79ca62a7d03948b11ac74f113aadeb3f3585fa30209b86aa052df9bd9ae4824724647991785e0bde62792e8432d5266dbdd7212432efebf71c95
+  metadata.gz: 42542453d2526f2a0e8fc10debc07648f80252e8c349ee9349eefecdd26756596da846a61c5f38e5b025b8c6eaae7ad1689737b953616504acfbd52c2771989b
+  data.tar.gz: aea66c00ee53155fe58367975988f2c450e8d9e366f4182a9a6268e036cf03a9235043dafc45812cb5122f55fe791a11cbabce5e10929fdea444e49ff824061c

data/.rubocop.yml

@@ -6,54 +6,24 @@ AllCops:
   Exclude:
     - 'gemfiles/vendor/**/*'
 
-# Broadly speaking, test code gets a pass for most of the Metrics family.
+# Some classes and methods need to be big.
 #
-# IMO test code is not the place get pedantic about class length,
-# method complexity, etc.  One should be encouraged to add more tests
-# with minimal friction, not forced to make a hard choice between
-# cutting tests or splitting up my test suites.
+# I'll break them down for testability or composability or
+# applicability, but not because I crossed an invisible line that
+# somebody made up.
 #
-Metrics/BlockLength:
-  Exclude:
-    - 'test/**/*.rb'
-Metrics/ClassLength:
-  Max: 400
-  Exclude:
-    - 'test/**/*.rb'
-
-# I like this Metric in principle, but I don't like the default max of
-# 15.
-#
-# Also, as per Metrics/ClassLength IMO this kind of limit should not
-# apply to test code (I get up to 318 over there).
-#
-Metrics/AbcSize:
-  Max: 30
-  Exclude:
-    - 'test/**/*.rb'
-
-# I like this Metric in principle, but I don't like the default max of
-# 10.
-#
-# Also, as per Metrics/ClassLength IMO this kind of limit should not
-# apply to test code.
-#
-Metrics/MethodLength:
-  Max: 50
-  Exclude:
-    - 'test/**/*.rb'
+Metrics:
+  Enabled: false
 
 # I put extra spaces in a lot of expressions for a lot of different
 # reasons, including especially readability.
 #
-# I reject these cops.
-#
 Layout:
   Enabled: false
 
 # As a group, the Style cops are bewilderingly opiniated.
 #
-# In some cases IMO they are harmful e.g. Style/TernaryParentheses.
+# In some cases IMO they are actively harmful e.g. Style/TernaryParentheses.
 #
 # I reject these cops.
 #

data/CHANGELOG.md

@@ -1,4 +1,11 @@
-## 0.1.3 (2019-06-067)
+## 0.1.4 (2019-11-18)
+- ICKUNLINK, as ICKDEL but uses Redis UNLINK for O(1) time.
+- Clear recommendation not to use backwash unless clearly needed.
+  - Assuming a full cset, it can more than double the cost of ICKEXCHANGE.
+- Performance optimizations in Redis Lua code.
+  - 13%-25% faster for large reservations on large Icks
+
+## 0.1.3 (2019-06-07)
 - Support for redis >= 4.0.0 added.
 - Breaking changes at redis v4.0.0 addressed.
 - Support for ruby < 2.2.2 dropped.

data/lib/redis/ick.rb

@@ -80,19 +80,47 @@ class Redis
     #
     # @param ick_key String the base key for the Ick
     #
+    # @param unlink true to use UNLINK, default false to use DEL.
+    #
     # @return an integer, the number of Redis keys deleted, which will
     # be >= 1 if an Ick existed at key.
     #
-    def ickdel(ick_key)
+    def ickdel(ick_key,unlink: false)
       if !ick_key.is_a?(String)
         raise ArgumentError, "bogus non-String ick_key #{ick_key}"
       end
-      _statsd_increment('profile.ick.ickdel.calls')
-      _statsd_time('profile.ick.ickdel.time') do
-        _eval(LUA_ICKDEL,ick_key)
+      stats_prefix = unlink ? 'profile.ick.ickunlink' : 'profile.ick.ickdel'
+      redis_cmd    = unlink ? 'UNLINK'                : 'DEL'
+      _statsd_increment("#{stats_prefix}.calls")
+      _statsd_time("#{stats_prefix}.time") do
+        _eval(
+          LUA_ICK_PREFIX +
+          "return redis.call('#{redis_cmd}',ick_key,ick_pset_key,ick_cset_key)",
+          ick_key
+        )
       end
     end
 
+    # Removes all data associated with the Ick in Redis at key.
+    #
+    # Similar to UNLINK key, http://redis.io/commands/unlink, but may
+    # unlink multiple keys which together implement the Ick data
+    # structure.
+    # 
+    # UNLINK is O(1) in the number of messages in the Ick, and is
+    # available with redis-server >= 4.0.0.  Physical space
+    # reclamation in Redis, which can be O(N), is deferred to
+    # asynchronous server threads.
+    #
+    # @param ick_key String the base key for the Ick
+    #
+    # @return an integer, the number of Redis keys unlinked, which will
+    # be >= 1 if an Ick existed at key.
+    #
+    def ickunlink(ick_key)
+      ickdel(ick_key,unlink: true)
+    end
+
     # Fetches stats.
     #
     # @param ick_key String the base key for the Ick
@@ -575,21 +603,6 @@ class Redis
       end
     }.freeze
 
-    #######################################################################
-    # LUA_ICKDEL
-    #######################################################################
-    #
-    # Removes all keys associated with the Ick at KEYS[1].
-    #
-    # @param uses no ARGV
-    #
-    # @return the number of Redis keys deleted, which will be 0 if and
-    # only if no Ick existed at KEYS[1]
-    #
-    LUA_ICKDEL = (LUA_ICK_PREFIX + %{
-      return redis.call('DEL',ick_key,ick_pset_key,ick_cset_key)
-    }).freeze
-
     #######################################################################
     # LUA_ICKSTATS
     #######################################################################
@@ -747,54 +760,76 @@ class Redis
     # cset by the commit function followed by up to ARGV[1] pairs
     # [member,score,...] from the reserve funciton.
     #
-    # Note: This Lua unpacks ARGV with the iterator ipairs() instead
-    # of unpack() to avoid a "too many results to unpack" failure at
-    # 8000 args.  However, the loop over many redis.call is
-    # regrettably heavy-weight.  From a performance standpoint it
-    # would be preferable to call ZREM in larger batches.
+    # Note: This Lua code calls unpack(ARGV,i,j) in limited-size
+    # slices, no larger than 7990, to avoid a "too many results to
+    # unpack" failure which has been observed when unpacking tables as
+    # small as 8000.
     #
     LUA_ICKEXCHANGE = (LUA_ICK_PREFIX + %{
       local reserve_size   = tonumber(ARGV[1])
       local backwash       = ARGV[2]
       local argc           = table.getn(ARGV)
       local num_committed  = 0
-      for i = 3,argc,1 do
-        local num_zrem     = redis.call('ZREM',ick_cset_key,ARGV[i])
-        num_committed      = num_committed + num_zrem
-      end
-      if 'backwash' == backwash then
-        local cset_all     = redis.call('ZRANGE',ick_cset_key,0,-1,'WITHSCORES')
-        local cset_size    = table.getn(cset_all)
-        for i = 1,cset_size,2 do
-          local member     = cset_all[i]
-          local score      = cset_all[i+1]
-          local old_score  = redis.call('ZSCORE',ick_pset_key,member)
-          if false == old_score then
-            redis.call('ZADD',ick_pset_key,score,member)
-          elseif tonumber(score) < tonumber(old_score) then
-            redis.call('ZADD',ick_pset_key,score,member)
+      local unpack_limit   = 7990
+      for i = 3,argc,unpack_limit do
+        local max        = math.min(i+unpack_limit,argc)
+        local num_zrem   = redis.call('ZREM',ick_cset_key,unpack(ARGV,i,max))
+        num_committed    = num_committed + num_zrem
+      end
+      local ick_fold = function(key_from,key_to,max_size_key_to)
+        while true do
+          local size_key_to       = redis.call('ZCARD',key_to)
+          local num               = math.min(
+            max_size_key_to - size_key_to,
+            unpack_limit / 2                 -- room for both scores and members
+          )
+          if num < 1 then
+            break
+          end
+          local head_from         =
+            redis.call('ZRANGE',key_from,0,num-1,'WITHSCORES')
+          local head_size         = table.getn(head_from)
+          if 0 == head_size then
+            break
+          end
+          local to_zadd           = {}       -- both scores and members
+          local to_zrem           = {}       -- members only
+          for i = 1,head_size,2 do
+            local member          = head_from[i]
+            local score_from      = tonumber(head_from[i+1])
+            local score_to        = redis.call('ZSCORE',key_to,member)
+            if false == score_to or score_from < tonumber(score_to) then
+              to_zadd[#to_zadd+1] = score_from
+              to_zadd[#to_zadd+1] = member
+            end
+            to_zrem[#to_zrem+1]   = member
+          end
+          redis.call('ZREM',key_from,unpack(to_zrem))
+          if 0 < table.getn(to_zadd) then
+            redis.call('ZADD',key_to,unpack(to_zadd))
           end
-        end
-        redis.call('DEL',ick_cset_key)
-      end
-      while true do
-        local cset_size    = redis.call('ZCARD',ick_cset_key)
-        if cset_size and reserve_size <= cset_size then
-          break
-        end
-        local first_pset   = redis.call('ZRANGE',ick_pset_key,0,0,'WITHSCORES')
-        if 0 == table.getn(first_pset) then
-          break
-        end
-        local first_member = first_pset[1]
-        local first_score  = tonumber(first_pset[2])
-        redis.call('ZREM',ick_pset_key,first_member)
-        local old_score    = redis.call('ZSCORE',ick_cset_key,first_member)
-        if false == old_score or first_score < tonumber(old_score) then
-          redis.call('ZADD',ick_cset_key,first_score,first_member)
         end
       end
+      if 'backwash' == backwash then
+        --
+        -- Fold everything in the cset back into the pset.
+        --
+        local pset_size = redis.call('ZCARD',ick_pset_key) or 0
+        local cset_size = redis.call('ZCARD',ick_cset_key) or 0
+        ick_fold(ick_cset_key,ick_pset_key,pset_size+cset_size)
+      end
+      --
+      -- Fold enough from the pset to the cset to grow the cset
+      -- to at most reserve_size members.
+      --
+      ick_fold(ick_pset_key,ick_cset_key,reserve_size)
+      --
+      -- Make sure ick_key exists per specification.
+      --
       redis.call('SETNX', ick_key, 'ick.v1')
+      --
+      -- Package up return results, which may be smaller than the cset.
+      --
       local result         = { num_committed }
       if reserve_size > 0 then
         local max          = reserve_size - 1

data/lib/redis/ick/version.rb

@@ -1,5 +1,5 @@
 class Redis
   class Ick
-    VERSION = '0.1.3'.freeze
+    VERSION = '0.1.4'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-ick
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - jhwillett
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-07 00:00:00.000000000 Z
+date: 2019-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis