redis-ick 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c22eec5b65c61b1bc5ec5c19a88d21da264df33
-  data.tar.gz: 0e7e7f500fc89fa0fb3394a8df7d4e8784c8e202
+  metadata.gz: cb3406be565b9f93bea296e10d6cb1d34c48707a
+  data.tar.gz: 92f6dd755dad1165b41dd2050547712124b0cbea
 SHA512:
-  metadata.gz: 1f9e5dda5e632a926420c6ce7c5483edf53f757718622d2ae2d151c7c46105c5a46e1c7f77ce06804a56ee60703e4b151a616c36dab5cf634569e9c789a1c4f7
-  data.tar.gz: d65b846e08e4c4ca477770c09ce9331fc6ed293971fb4b0bd5ef1139c6020e36515a10bcd4971c633f47b438489199d1fb2e3329b112f102f127528d71f6700c
+  metadata.gz: d811a5bc57c88d9833ca04c0e1715e1dbeb2a6be1de01bc6a41dec3a06d009a11a9e5884b0a8ec01b75c0e6cdaacdaeb284c5b3d678ebf071bc54812691a1935
+  data.tar.gz: 1bd851eadaff3a9fe3dc1996eb38654121dd4785857ea3fc14c9e2cabfe7b815273091f1b677ae9720e78265ee3a382534cf739c6666a3d34bcb86421bbeafbb

data/.rubocop.yml

@@ -1,4 +1,3 @@
-
 AllCops:
   Include:
     - Rakefile
@@ -50,103 +49,16 @@ Metrics/MethodLength:
 Layout:
   Enabled: false
 
-# I like a lot of the Lint tests, but not these.
-#
-Lint/AmbiguousBlockAssociation:           # obnoxiously rejects idiomatic Ruby
-  Enabled: false
-
-# Ick#ickreserve returns an Array of Arrays.
-#
-# Performance/HashEachMethods complains that:
-#
-#   ick.ickreserve('key',num).each do |message,score|
-#     ...
-#   end
-#
-# would be more efficient as a each_key loop.
-#
-# This cop inappropriately requires us to change working code to
-# broken code.  This is a known issue which the authors of Rubocop do
-# not intend to change:
-#
-#   https://github.com/bbatsov/rubocop/issues/4732
-#
-# My reaction is to blacklist this cop.
+# As a group, the Style cops are bewilderingly opiniated.
 #
-Performance/HashEachMethods:
-  Enabled: false
-
-# This does no more than insist I type "format" instead of "sprintf",
-# where the two are aliases.
-#
-Style/FormatString:
-  Enabled: false
-
-# There is nothing wrong with Ruby 1.9 Hash syntax.
-#
-Style/HashSyntax:
-  Enabled: false
-
-# No.  Indeed, postfix if can often drive a line over 80 columns wide.
-#
-Style/IfUnlessModifier:
-  Enabled: false
-
-# No.  There is nothing wrong with "if !foo".
+# In some cases IMO they are harmful e.g. Style/TernaryParentheses.
 #
-# As far as I'm concerned, "unless" is in poor taste because it means
-# I have to think in English in two different logical senses - and
-# English is a poor language for logical senses.
-#
-Style/NegatedIf:
-  Enabled: false
-
-# "Do not use semicolons to terminate expressions."
-#
-# That's great when I terminate a single-line expression with a
-# redundant semicolo because I forget I'm not using C.
-#
-# But when I'm using a semicolon to separate two expressions there is
-# no other choice.  So this really ought to be Style/OneExprPerLine,
-# which I reject.
-#
-Style/Semicolon:
-  Enabled: false
-
-# No.
-#
-# Some lines must have '\"'.  It is ugly to use a mix of '"' and '\''
-# in LoCs which are close to one another.  Therefore, banning '"' if
-# '"' is not strictly necessary drives visual inconsistency.
-#
-Style/StringLiterals:
-  Enabled: false
-
-# This is the same kind of obnoxious pedantry which drove Hungarian
-# Notation.
-#
-# The [] literal syntax is perfectly servicable and there is no point
-# _tightly_ coupling it to the content of the array.  That's why we
-# have context-free grammers!
-#
-Style/SymbolArray:
-  Enabled: false
-
-# Shockingly, this cop requires us to *OMIT*, not *INCLUDE* parens in
-# ternery conditons.
-#
-# IMO this one is actively harmful in that it discourages attention to
-# clarity and avoiding some nasty precedence surprises.
+# I reject these cops.
 #
-Style/TernaryParentheses:
+Style:
   Enabled: false
 
-# I am a huge fan of using trailing commas when I break an argument
-# list down one-per line.
-#
-# As such, I reject these tests.
+# I like a lot of the Lint tests, but not these.
 #
-Style/TrailingCommaInArguments:
-  Enabled: false
-Style/TrailingCommaInLiteral:
+Lint/AmbiguousBlockAssociation:           # obnoxiously rejects idiomatic Ruby
   Enabled: false

data/.travis.yml

@@ -10,7 +10,7 @@
 sudo: true
 language: ruby
 before_install:
-  - gem install bundler -v 1.14.6
+  - gem install bundler -v 1.16.1
   - sudo add-apt-repository -y ppa:twemproxy/stable
   - sudo apt-get update -y
   - sudo apt-get install -y twemproxy
@@ -18,6 +18,9 @@ services:
   - redis-server
 rvm:
   - 2.1.6
+  - 2.2.9
+  - 2.4.3
+  - 2.5.0
 before_script:
   - bundle exec rubocop --version
   - nutcracker --version
@@ -25,6 +28,6 @@ before_script:
   - nutcracker             --conf-file=.travis/nutcracker.yml &
   - sleep 0.3
 script:
-  - bundle exec rubocop
+  - bundle exec rubocop --display-cop-names --display-style-guide
   - bundle exec env REDIS_URL=redis://localhost:6379 rake test
   - bundle exec env REDIS_URL=redis://localhost:22121 rake test

data/CHANGELOG.md

@@ -0,0 +1,31 @@
+## 0.1.0 (2018-03-20)
+
+- Added ickexchange which combines ickcommit+ickreserve.
+- Introduced backwash to ickreserve and ickexchange.
+- Expanded .travis.yml to cover more rvm versions.
+- Shrink Rubocop coverage to exclude `Style/*`.
+- Moves version history out into CHANGELOG.md.
+
+## 0.0.5 (2017-09-20)
+
+- Rework ickstats so it no longer angers Twemproxy, per https://github.com/ProsperWorks/redis-ick/issues/3, by producing a nested Array response.
+
+## 0.0.4 (2017-09-12)
+
+- Imported text from original design doc to README.md, polish.
+- Rubocop polish and defiance.
+- Development dependency on [redis-key_hash](https://github.com/ProsperWorks/redis-key_hash) to test prescriptive hash claims.
+- Identified limits of prescriptive hash robustness.
+
+## 0.0.3 (2017-08-29)
+- Got .travis.yml working with a live redis-server.
+- Runtime dependency on redis-script_manager for Ick._eval.
+- Initial Rubocop integration.
+- Misc cleanup.
+
+## 0.0.2 (2017-08-29)
+- Broke out into Prosperworks/redis-ick, make public.
+
+## 0.0.1 (prehistory)
+- Still in Prosperworks/ALI/vendor/gems/redis-ick.
+

data/README.md

@@ -44,10 +44,10 @@ of dirty _documents_ per unit time, not in the number of dirty
 _operations_ per unit time.  In a sense, the more we fall behind the
 slower we fall.
 
-**Big Problem**: The Forgotten Dirtiness Problem.  If some document is
-dirtied a second time after the start of process_batch_slowly(), when
-process_batch_slowly() end we will drop that document from the queue.
-Thus, the document will be dirty but no longer in the queue!
+**Big Problem**: The Forgotten Dirtiness Problem.  A document could be
+dirtied a second time while process_batch_slowly() is processing it.
+When we we drop that document from the queue, we may end up with the
+document still dirty but no longer in the queue.
 
 **Small Problem**: The Hot Data Starvation Problem.  Because we score
 by time-of-dirtiness and we use ZRANGEBYRANK starting at 0, each batch
@@ -81,13 +81,13 @@ taken after process_batch_slowly().  Only documents whose timestamps
 did not change between the two snapshots are removed from the queue.
 
 Notice how the critical section no longer includes
-process_batch_slowly().  Instead it only spans two Redis ops and some
-local set arithmetic which.
+process_batch_slowly().  Instead it only spans two Redis ops and a bit
+of local set arithmetic.
 
-The critical section and the Forgotten Dirtiness Problem which it
-causes is still there, but is much smaller.  In practice we have
-process_batch_slowly() taking minutes, but even in extreme situations
-this critical section never took more than 3 seconds.
+The critical section which causes the Forgotten Dirtiness Problem is
+still there, but is much smaller.  In practice we see
+process_batch_slowly() take minutes, but even in extreme situations
+this smaller critical section never takes more than 3 seconds.
 
 ## Proposal: The Ick Pattern
 
@@ -96,14 +96,17 @@ identified the Hot Data Starvation Problem.  We developed Ick and
 switched to this almost familiar pattern:
 
     # in any process: whenever a document is dirtied
-    Ick.ickadd(redis,queue_key,Time.now.to_f,document_id)
-
-    # in the indexer process:
-    batch = Ick.ickreserve(redis,queue_key,batch_size)
-    process_batch_slowly(batch)
-    # burn down the queue only if the batch succeeded
-    Ick.ickcommit(redis,queue_key,*members_of(batch))   # critical section gone
+    #
+    Ick.new(redis).ickadd(queue_key,Time.now.to_f,document_id)
 
+    # in the indexer process, burn down the queue only if the batch succeeded:
+    #
+    ick     = Ick.new(redis)
+    while still_going() do
+      batch = ick.ickreserve(queue_key,batch_size)
+      process_batch_slowly(batch)
+      ick.ickcommit(queue_key,*members_of(batch))  # critical section gone
+    end
 
 Ick solves for failover via a two phase commit protocol between
 **ickreserve** and **ickcommit**.  If there is a failure during
@@ -117,23 +120,36 @@ only ever called from the indexer and producers do not mutate the
 consumer set.
 
 Ick solves the Hot Data Starvation Problem by a subtle change in
-ICKADD.  Unlike ZADD, which overwrites the old score when a message is
-re-added, or ZADD NX which always preserves the old score, ICKADD
-always takes the _min_ of the old and new scores.  Thus, Ick tracks
-the first-known ditry time for a message even when there is time skew
-in the producers.  The longer entries stay in the consumer set, the
-more they implicitly percolate toward the cold end regardless of how
-many updates they receive.  Ditto in the consumer set.  Provided that
-all producers make a best effort to use only current or future
-timestamps when they call ICKADD, the ICKRESERVE batch will always
-include the oldest entries and there will be no starvation.
+**ickadd**.  Unlike ZADD, which overwrites the old score when a
+message is re-added, or ZADD NX which always preserves the old score,
+**ickadd** always takes the _min_ of the old and new scores.  Thus,
+Ick tracks the first-known ditry time for a message even when there is
+time skew in the producers.  The longer entries stay in the consumer
+set, the more they implicitly percolate toward the cold end regardless
+of how many updates they receive.  Ditto in the consumer set.
+Provided that all producers make a best effort to use only current or
+future timestamps when they call **ickadd*, the **ickreserve** batch
+will always include the oldest entries and there will be no
+starvation.
+
+To reduce Redis round-trips, Ick also supports an operation
+**ickexchange** which combines **ickcommit** with **ickreserve**
+
+    ick     = Ick.new(redis)
+    batch   = []
+    while still_going() do
+      batch = ick.ickexchange(queue_key,batch_size,*batch) # commit + reserve
+      process_batch_slowly(batch)
+    end
+    ick.ickexchange(queue_key,0,*batch)                    # commit final batch
+
 
 Apology: I know that [Two-Phase
 Commit](https://en.wikipedia.org/wiki/Two-phase_commit_protocol) has a
 different technical meaning than what Ick does.  Unfortunately I can't
-find a better name for this very common failsafe queue pattern.  I
-suppose we could think of the Redis sorted set as the coordinator and
-the consumer process as the (single) participant node and, generously,
+find a better name for this common fail-safe pattern.  I suppose we
+could think of the Redis sorted set as the coordinator and the
+consumer process as the (single) participant node and, generously,
 Two-Phase Commit might be taken to describe Ick.
 
 
@@ -146,7 +162,7 @@ An Ick is a collection of three Redis keys which all live on the same
 * producer set, a sorted set into which we flag keys as dirty with timestamps
 * consumer set, a sorted set from which the indexer pulls batches to index
 
-### Ick defines 5 operations on this data via Lua on Redis:
+### Ick defines 6 operations on this data via Lua on Redis:
 
 * **ickdel**: removes all keys associated with a given Ick structure
 * **ickstats**: returns a hash of stats including version and size
@@ -159,24 +175,32 @@ An Ick is a collection of three Redis keys which all live on the same
   * when a member-is re-added it takes the lowest of 2 scores
   * returns the results as an array
 * **ickcommit**: deletes members from the consumer set
+* **ickexchange**: combines **ickcommit** and **ickreserve** in one op
 
 Reminder: With few exceptions, all Redis commands are atomic and
 transactional.  This includes any Lua scripts such as those which
 implement Ick.  This atomicity guarantee is important to the
 correctness of Ick, but because it is inherent in Redis/Lua, does not
-appear explicitly in any of the Ick sources.
+appear explicitly in any of the Ick source.
 
 ## Fabulous Diagram
 
 Here is a coarse dataflow for members moving through an Ick.
 
-    app
-    |
-    +-- **ickadd** --> producer set
-                   |
-                   +-- **ickreserve** --> consumer set
-                                      |
-                                      +-- **ickcommit** --> forgotten
+    **ickadd**     --> producer set    --+
+                                         |
+    **ickreserve**     consumer set -----+--> consumer set --+
+                                                             |
+    **ickcommit**                                            +--> forgotten
+
+Here is the dataflow for members moving through an Ick with backwash.
+
+    **ickadd**     --> producer set --+--+
+                                      |  |
+    **ickreserve**     consumer set --+  +--> consumer set --+
+                                                             |
+    **ickcommit**                                            +--> forgotten
+
 
 ## Miscellanea
 
@@ -186,10 +210,10 @@ Even though one Ick uses three Redis keys, Ick is compatible with
 Redis Cluster.  At ProsperWorks we use it with RedisLabs Enterprise
 Cluster.
 
-Ick does some very tricky things to compute the producer set and
-consumer set keys from the master key in a way which puts them all on
-the same slot in both Redis Cluster and with RLEC's default
-prescriptive hashing algorithm.
+Ick does tricky things to compute the producer set and consumer set
+keys from the master key in a way which puts them all on the same slot
+in both Redis Cluster and with RLEC's default prescriptive hashing
+algorithm.
 
 See [redis-key_hash](https://github.com/ProsperWorks/redis-key_hash)
 for how test this.
@@ -217,42 +241,45 @@ when one consumer halts?  How do we prevent the cold end of the
 producer set from getting clogged up with messages destined for the
 idle consumer?
 
-We prefer handling those issues in higher-level code.  Thus, Ick by
-itself does not attempt to solve scalability.
+We prefer handling those issues in higher-level code.  Ick by itself
+does not attempt to solve scalability.
 
 
 ### Some Surprises Which Can Be Gotchas in Test
 
-Because ICKADD uses write-folding semantics over the producer set,
-ICKADD might or might not grow the total size of the queue.
+Because **ickadd** uses write-folding semantics over the producer set,
+**ickadd** might or might not grow the total size of the queue.
 
-ICKRESERVE is not a read-only operation.  It can mutate both the
-producer set and the consumer set.  Because ICKRESERVE uses
+**ickreserve** is not a read-only operation.  It can mutate both the
+producer set and the consumer set.  Because **ickreserve** uses
 write-folding semantics between the producer set and the consumer set,
-ICKRESERVE(N) might:
+`ickreserve(ick_key,N)` might:
 
 * shrink the producer set by N and grow the consumer set by N
 * shrink the producer set by 0 and grow the consumer set by 0
 * shrink the producer set by N and grow the consumer set by 0
-* or anything in between
+* anything where the producer set shrinks at least as much as consumer set grows
 
 Because Ick always uses the min when multiple scores are present for
-one message, ICKADD can rearrange the order of the producer set and
-ICKRESERVE can rearrange the order of the consumer set in surprising
-ways.
-
-ICKADD write-folds in the producer set but not in the consumer set.
-Thus, one message can appear in both the producer set and the consumer
-set.  At first this seems wrong and inefficient, but in fact it is a
-desirable property.  When a message is in both sets, it means it was
-included in a batch by ICKRESERVE, then added by ICKADD, but has yet
-to be ICKCOMMITed.  The interpretation for this is that the consumer
-is actively engaged in updating the downstream systems.  But that
-means, at the Ick it is indeterminate whether the message is still
-dirty or has been cleaned.  That is, being in both queues corresponds
-exactly to a message being in the critical section where a race
-condition is possible.  Thus, we _want_ it to still be dirty and to
-appear in a future batch.
+one message, **ickadd** can rearrange the order of the producer set
+and **ickreserve** can rearrange the order of the consumer set in
+surprising ways.
+
+With backwash enabled, **ickreserve** can result in a complete
+exchange of entries between the producer set and the consumer set.
+
+**ickadd** write-folds in the producer set but not in the consumer
+set.  Thus, one message can appear in both the producer set and the
+consumer set.  At first this seems wrong and inefficient, but in fact
+it is a desirable property.  When a message is in both sets, it means
+it was included in a batch by **ickreserve**, then added by
+**ickadd**, but has yet to be **ickcommit**-ed.  The interpretation
+for this is that the consumer is actively engaged in updating the
+downstream systems.  But that means, at the Ick it is indeterminate
+whether the message is still dirty or has been cleaned.  That is,
+being in both queues corresponds exactly to a message being in the
+critical section where a race condition is possible.  Thus, we _want_
+it to still be dirty and to appear in a future batch.
 
 None of these surprises is a bug in Ick: they are all consistent with
 the design and the intent.  But they are surprises nonetheless and can

data/lib/redis/ick/version.rb

@@ -1,38 +1,5 @@
 class Redis
   class Ick
-    #
-    # Version plan/history:
-    #
-    # 0.0.1 - Still in Prosperworks/ALI/vendor/gems/redis-ick.
-    #
-    # 0.0.2 - Broke out into Prosperworks/redis-ick, make public.
-    #
-    # 0.0.3 - Got .travis.yml working with a live redis-server.
-    #
-    #         Runtime dependency on redis-script_manager for
-    #         Ick._eval.
-    #
-    #         Initial Rubocop integration.
-    #
-    #         Misc cleanup.
-    #
-    # 0.0.4 - Imported text from original design doc to README.md, polish.
-    #
-    #         Rubocop polish and defiance.
-    #
-    #         Development dependency on redis-key_hash to test
-    #         prescriptive hash claims.  Identified limits of
-    #         prescriptive hash robustness.
-    #
-    # 0.0.5 - Rework ickstats so it no longer angers Twemproxy, per
-    #         https://github.com/ProsperWorks/redis-ick/issues/3,
-    #         by producing a nested Array response.
-    #
-    # 0.1.0 - (future) Big README.md and Rdoc update, solicit feedback
-    #         from select external beta users.
-    #
-    # 0.2.0 - (future) Incorporate feedback, announce.
-    #
-    VERSION = '0.0.5'.freeze
+    VERSION = '0.1.0'.freeze
   end
 end

data/lib/redis/ick.rb

@@ -3,7 +3,7 @@ require 'redis/script_manager'
 
 class Redis
 
-  # Binds Lua code to provide the Ick operations in Redis.
+  # Accessor for Ick data structures in Redis.
   #
   class Ick
 
@@ -107,51 +107,38 @@ class Redis
         raise ArgumentError, "bogus non-String ick_key #{ick_key}"
       end
       _statsd_increment('profile.ick.ickstats.calls')
-      raw_ickstats_results = nil
+      raw_results = nil
       _statsd_time('profile.ick.time.ickstats') do
-        raw_ickstats_results = _eval(LUA_ICKSTATS,ick_key)
-      end
-      if raw_ickstats_results.is_a?(Redis::Future)
-        #
-        # We extend the Redis::Future with a continuation so we can add
-        # our own post-processing.
-        #
-        class << raw_ickstats_results
-          alias_method :original_value, :value
-          def value
-            ::Redis::Ick._postprocess_ickstats_results(original_value)
+        raw_results = _eval(LUA_ICKSTATS,ick_key)
+      end
+      _postprocess(
+        raw_results,
+        lambda do |results|
+          return nil if !results
+          #
+          # LUA_ICKSTATS returned bulk data response [k,v,k,v,...]
+          #
+          stats = Hash[*results]
+          #
+          # From http://redis.io/commands/eval, the "Lua to Redis conversion
+          # table" states that:
+          #
+          #   Lua number -> Redis integer reply (the number is converted
+          #   into an integer)
+          #
+          #   ...If you want to return a float from Lua you should return
+          #   it as a string.
+          #
+          # LUA_ICKSTATS works around this by converting certain stats to
+          # strings.  We reverse that conversion here.
+          #
+          stats.keys.select{|k|/_min$/ =~ k || /_max$/ =~ k}.each do |k|
+            next if !stats[k]
+            stats[k] = (/^\d+$/ =~ stats[k]) ? stats[k].to_i : stats[k].to_f
           end
+          stats
         end
-        raw_ickstats_results
-      else
-        ::Redis::Ick._postprocess_ickstats_results(raw_ickstats_results)
-      end
-    end
-
-    def self._postprocess_ickstats_results(raw_ickstats_results)
-      return nil if !raw_ickstats_results
-      #
-      # LUA_ICKSTATS returned bulk data response [k,v,k,v,...]
-      #
-      stats = Hash[*raw_ickstats_results]
-      #
-      # From http://redis.io/commands/eval, the "Lua to Redis conversion
-      # table" states that:
-      #
-      #   Lua number -> Redis integer reply (the number is converted
-      #   into an integer)
-      #
-      #   ...If you want to return a float from Lua you should return
-      #   it as a string.
-      #
-      # LUA_ICKSTATS works around this by converting certain stats to
-      # strings.  We reverse that conversion here.
-      #
-      stats.keys.select{|k|/_min$/ =~ k || /_max$/ =~ k}.each do |k|
-        next if !stats[k]
-        stats[k] = (/^\d+$/ =~ stats[k]) ? stats[k].to_i : stats[k].to_f
-      end
-      stats
+      )
     end
 
     # Adds all the specified members with the specified scores to the
@@ -229,11 +216,16 @@ class Redis
     #
     # @param max_size max number of messages to reserve
     #
+    # @param backwash if true, in the reserve function cset members
+    # with high scores are swapped out for pset members with lower
+    # scores.  Otherwise cset members remain in the cset until
+    # committed regardless of how low scores in the pset might be.
+    #
     # @return a list of up to max_size pairs, similar to
     # Redis.current.zrange() withscores: [ member_string, score_number ]
     # representing the lowest-scored elements from the producer set.
     #
-    def ickreserve(ick_key,max_size=0)
+    def ickreserve(ick_key,max_size=0,backwash: false)
       if !ick_key.is_a?(String)
         raise ArgumentError, "bogus non-String ick_key #{ick_key}"
       end
@@ -245,36 +237,16 @@ class Redis
       end
       _statsd_increment('profile.ick.ickreserve.calls')
       _statsd_timing('profile.ick.ickreserve.max_size',max_size)
-      raw_ickreserve_results = nil
+      raw_results   = nil
       _statsd_time('profile.ick.time.ickreserve') do
-        raw_ickreserve_results =
-          _eval(
-            LUA_ICKRESERVE,
-            ick_key,
-            max_size
-          )
-      end
-      if raw_ickreserve_results.is_a?(Redis::Future)
-        #
-        # We extend the Redis::Future with a continuation so we can
-        # add our own post-processing.
-        #
-        class << raw_ickreserve_results
-          alias_method :original_value, :value
-          def value
-            original_value.each_slice(2).map do |p|
-              [ p[0], ::Redis::Ick._floatify(p[1]) ]
-            end
-          end
-        end
-        raw_ickreserve_results
-      else
-        results = raw_ickreserve_results.each_slice(2).map do |p|
-          [ p[0], ::Redis::Ick._floatify(p[1]) ]
-        end
-        _statsd_timing('profile.ick.ickreserve.num_results',results.size)
-        results
-      end
+        raw_results = _eval(
+          LUA_ICKEXCHANGE,
+          ick_key,
+          max_size,
+          backwash ? 'backwash' : false,
+        )
+      end
+      _postprocess(raw_results,Skip0ThenFloatifyPairs)
     end
 
     # Removes the indicated members from the producer set, if present.
@@ -301,8 +273,169 @@ class Redis
       end
       _statsd_increment('profile.ick.ickcommit.calls')
       _statsd_timing('profile.ick.ickcommit.members',members.size)
+      raw_results = nil
       _statsd_time('profile.ick.time.ickcommit') do
-        _eval(LUA_ICKCOMMIT,ick_key,*members)
+        raw_results = _eval(
+          LUA_ICKEXCHANGE,
+          ick_key,
+          0,
+          false,              # backwash not relevant in ickcommit
+          *members
+        )
+      end
+      # 
+      # raw_results are num_committed followed by 0 message-and-score
+      # pairs.
+      #
+      # We just capture the num_committed.
+      #
+      _postprocess(raw_results,lambda { |results| results[0] })
+    end
+
+    # ickexchange combines several functions in one Redis round-trip.
+    #
+    # 1. As ickcommit, removes consumed members from the consumer set.
+    #
+    # 2. As ickreserve, tops up the consumer set from the producer and
+    #    returns the requested new consumer members, if any.
+    #
+    # @param ick_key String the base key for the Ick
+    #
+    # @param reserve_size Integer max number of messages to reserve.
+    #
+    # @param commit_members Array members to be committed.
+    #
+    # @param backwash if true, in the reserve function cset members
+    # with high scores are swapped out for pset members with lower
+    # scores.  Otherwise cset members remain in the cset until
+    # committed regardless of how low scores in the pset might be.
+    #
+    # @return a list of up to reserve_size pairs, similar to
+    # Redis.current.zrange() withscores: [ message, score ]
+    # representing the lowest-scored elements from the producer set
+    # after the commit and reserve operations.
+    #
+    def ickexchange(ick_key,reserve_size,*commit_members,backwash: false)
+      if !ick_key.is_a?(String)
+        raise ArgumentError, "bogus non-String ick_key #{ick_key}"
+      end
+      if !reserve_size.is_a?(Integer)
+        raise ArgumentError, "bogus non-Integer reserve_size #{reserve_size}"
+      end
+      if reserve_size < 0
+        raise ArgumentError, "bogus negative reserve_size #{reserve_size}"
+      end
+      _statsd_increment('profile.ick.ickexchange.calls')
+      _statsd_timing('profile.ick.ickexchange.reserve_size',reserve_size)
+      _statsd_timing(
+        'profile.ick.ickexchange.commit_members',
+        commit_members.size
+      )
+      raw_results = nil
+      _statsd_time('profile.ick.time.ickexchange') do
+        raw_results = _eval(
+          LUA_ICKEXCHANGE,
+          ick_key,
+          reserve_size,
+          backwash ? 'backwash' : false,
+          commit_members
+        )
+      end
+      _postprocess(raw_results,Skip0ThenFloatifyPairs)
+    end
+
+    # Postprocessing done on the LUA_ICKEXCHANGE results for both
+    # ickreserve and ickexchange.
+    #
+    # results are num_committed followed by N message-and-score
+    # pairs.
+    #
+    # We do results[1..-1] to skip the first element, num_committed.
+    #
+    # On the rest, we floatify the scores to convert from Redis
+    # number-as-string limitation to Ruby Floats.
+    #
+    # This is similar to to Redis::FloatifyPairs:
+    #
+    # https://github.com/redis/redis-rb/blob/master/lib/redis.rb#L2887-L2896
+    #
+    Skip0ThenFloatifyPairs = lambda do |results|
+      results[1..-1].each_slice(2).map do |m_and_s|
+        [ m_and_s[0], ::Redis::Ick._floatify(m_and_s[1]) ]
+      end
+    end
+
+    # Calls back to block with the results.
+    # 
+    # If raw_results is a Redis::Future, callback will be deferred
+    # until the future is expanded.
+    #
+    # Otherwise, callback will happen immediately.
+    #
+    def _postprocess(raw_results,callback)
+      if raw_results.is_a?(Redis::Future)
+        #
+        # Redis::Future have a built-in mechanism for calling a
+        # transformation on the raw results.
+        #
+        # Here, we monkey-patch not the Redis::Future class, but just
+        # this one raw_results object.  We give ourselves a door to
+        # set the post-processing transformation.
+        #
+        # The transformation will be called only once when the real
+        # results are materialized.
+        #
+        class << raw_results
+          def transformation=(transformation)
+            raise "transformation collision" if @transformation
+            @transformation = transformation
+          end
+        end
+        raw_results.transformation = callback
+        raw_results
+      else
+        #
+        # If not Redis::Future, we invoke the callback immediately.
+        #
+        callback.call(raw_results)
+      end
+    end
+
+    # A deferred computation which allows us to perform post-processing
+    # on results which come back from redis pipelines.
+    #
+    # The idea is to regain some measure of composability by allowing
+    # utility methods to respond polymorphically depending on whether
+    # they are called in a pipeline.
+    #
+    # TODO: Where this utility lives in the code is not very well
+    # thought-out.  This is more broadly applicable than just for
+    # Icks.  This probably belongs in its own file, or in RedisUtil,
+    # or as a monkey-patch into redis-rb.  This is intended for use
+    # with Redis::Futures, but has zero Redis-specific code.  This is
+    # more broadly applicable, maybe, than Redis. This is in class Ick
+    # for the time being only because Ick.ickstats() is where I first
+    # needed this and it isn't otherwise obvious where to put this.
+    #
+    class FutureContinuation
+      #
+      # The first (and only the first) time :value is called on this
+      # FutureContinuation, conversion will be called.
+      #
+      def initialize(continuation)
+        @continuation = continuation
+        @result       = nil
+      end
+      #
+      # Force the computation.  :value is chosen as the name of this
+      # method to be duck-typing compatible with Redis::Future.
+      #
+      def value
+        if @continuation
+          @result       = @continuation.call
+          @continuation = nil
+        end
+        @result
       end
     end
 
@@ -310,7 +443,9 @@ class Redis
     # etc.
     #
     # So we can be certain of compatibility, this was stolen with tweaks
-    # from https://github.com/redis/redis-rb/blob/master/lib/redis.rb.
+    # from:
+    #
+    #   https://github.com/redis/redis-rb/blob/master/lib/redis.rb#L2876-L2885
     #
     def self._floatify(str)
       raise ArgumentError, "not String: #{str}" if !str.is_a?(String)
@@ -569,9 +704,20 @@ class Redis
     }).freeze
 
     #######################################################################
-    # LUA_ICKRESERVE
+    # LUA_ICKEXCHANGE: commit then reserve
     #######################################################################
     #
+    # Commit Function
+    #
+    # Removes specified members in ARGV[2..N] from the pset, then tops
+    # up the cset to up to size ARGV[1] by shifting the lowest-scored
+    # members over from the pset.
+    #
+    # The cset might already be full, in which case we may shift fewer
+    # than ARGV[1] elements.
+    #
+    # Reserve Function
+    #
     # Tops up the cset to up to size ARGV[1] by shifting the
     # lowest-scored members over from the pset.
     #
@@ -582,66 +728,75 @@ class Redis
     # are duplicate messages, we may remove more members from the pset
     # than we add to the cset.
     #
-    # @param ARGV a single number, batch_size, the desired
-    # size for cset and to be returned
+    # @param ARGV[1] single number, batch_size, the desired size for
+    # cset and to be returned
+    #
+    # @param ARGV[2] string, 'backwash' for backwash
     #
-    # @return a bulk response, up to ARGV[1] pairs [member,score,...]
+    # @param ARGV[3..N] messages to be removed from the cset before reserving
     #
-    LUA_ICKRESERVE = (LUA_ICK_PREFIX + %{
-      local target_cset_size = tonumber(ARGV[1])
+    # @return a bulk response, the number of members removed from the
+    # cset by the commit function followed by up to ARGV[1] pairs
+    # [member,score,...] from the reserve funciton.
+    #
+    # Note: This Lua unpacks ARGV with the iterator ipairs() instead
+    # of unpack() to avoid a "too many results to unpack" failure at
+    # 8000 args.  However, the loop over many redis.call is
+    # regrettably heavy-weight.  From a performance standpoint it
+    # would be preferable to call ZREM in larger batches.
+    #
+    LUA_ICKEXCHANGE = (LUA_ICK_PREFIX + %{
+      local reserve_size   = tonumber(ARGV[1])
+      local backwash       = ARGV[2]
+      local argc           = table.getn(ARGV)
+      local num_committed  = 0
+      for i = 3,argc,1 do
+        local num_zrem     = redis.call('ZREM',ick_cset_key,ARGV[i])
+        num_committed      = num_committed + num_zrem
+      end
+      if 'backwash' == backwash then
+        local cset_all     = redis.call('ZRANGE',ick_cset_key,0,-1,'WITHSCORES')
+        local cset_size    = table.getn(cset_all)
+        for i = 1,cset_size,2 do
+          local member     = cset_all[i]
+          local score      = cset_all[i+1]
+          local old_score  = redis.call('ZSCORE',ick_pset_key,member)
+          if false == old_score then
+            redis.call('ZADD',ick_pset_key,score,member)
+          elseif score < tonumber(old_score) then
+            redis.call('ZADD',ick_pset_key,score,member)
+          end
+        end
+        redis.call('DEL',ick_cset_key)
+      end
       while true do
-        local ick_cset_size  = redis.call('ZCARD',ick_cset_key)
-        if ick_cset_size and target_cset_size <= ick_cset_size then
+        local cset_size    = redis.call('ZCARD',ick_cset_key)
+        if cset_size and reserve_size <= cset_size then
           break
         end
-        local first_in_pset  = 
-          redis.call('ZRANGE',ick_pset_key,0,0,'WITHSCORES')
-        if 0 == table.getn(first_in_pset) then
+        local first_pset   = redis.call('ZRANGE',ick_pset_key,0,0,'WITHSCORES')
+        if 0 == table.getn(first_pset) then
           break
         end
-        local first_member   = first_in_pset[1]
-        local first_score    = tonumber(first_in_pset[2])
+        local first_member = first_pset[1]
+        local first_score  = tonumber(first_pset[2])
         redis.call('ZREM',ick_pset_key,first_member)
-        local old_score      = redis.call('ZSCORE',ick_cset_key,first_member)
+        local old_score    = redis.call('ZSCORE',ick_cset_key,first_member)
         if false == old_score or first_score < tonumber(old_score) then
           redis.call('ZADD',ick_cset_key,first_score,first_member)
         end
       end
       redis.call('SETNX', ick_key, 'ick.v1')
-      if target_cset_size <= 0 then
-        return {}
-      else
-        local max            = target_cset_size - 1
-        return redis.call('ZRANGE',ick_cset_key,0,max,'WITHSCORES')
-      end
-    }).freeze
-
-    #######################################################################
-    # LUA_ICKCOMMIT
-    #######################################################################
-    #
-    # Removes specified members from the pset.
-    #
-    # @param ARGV a list of members to be removed from the cset
-    #
-    # @return the number of members removed
-    #
-    # Note: This this Lua unpacks ARGV with the iterator ipairs()
-    # instead of unpack() to avoid a "too many results to unpack"
-    # failure at 8000 args.  However, the loop over many redis.call is
-    # regrettably heavy-weight.  From a performance standpoint it
-    # would be preferable to call ZREM in larger batches.
-    #
-    LUA_ICKCOMMIT = (LUA_ICK_PREFIX + %{
-      redis.call('SETNX', ick_key, 'ick.v1')
-      if 0 == table.getn(ARGV) then
-        return 0
-      end
-      local num_removed = 0
-      for i,v in ipairs(ARGV) do
-        num_removed = num_removed + redis.call('ZREM',ick_cset_key,v)
+      local result         = { num_committed }
+      if reserve_size > 0 then
+        local max          = reserve_size - 1
+        local cset_batch   =
+          redis.call('ZRANGE',ick_cset_key,0,max,'WITHSCORES')
+        for _i,v in ipairs(cset_batch) do
+          table.insert(result,v)
+        end
       end
-      return num_removed
+      return result
     }).freeze
 
   end

data/redis-ick.gemspec

@@ -21,14 +21,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler',              '~> 1.14'
-  spec.add_development_dependency 'rake',                 '~> 10.0'
-  spec.add_development_dependency 'minitest',             '~> 5.0'
+  spec.add_development_dependency 'bundler',              '~> 1.16.1'
+  spec.add_development_dependency 'minitest',             '~> 5.11.3'
+  spec.add_development_dependency 'rake',                 '~> 12.3.1'
   spec.add_development_dependency 'redis-key_hash',       '~> 0.0.4'
   spec.add_development_dependency 'redis-namespace',      '~> 1.5'
-  spec.add_development_dependency 'rubocop',              '~> 0.50.0'
+  spec.add_development_dependency 'rubocop',              '~> 0.54.0'
 
-  spec.add_dependency             'redis',                '~> 3.2'
+  spec.add_runtime_dependency     'redis',                '~> 3.2'
   spec.add_runtime_dependency     'redis-script_manager', '~> 0.0.2'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-ick
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - jhwillett
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-20 00:00:00.000000000 Z
+date: 2018-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,42 +16,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: 1.16.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: 1.16.1
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 5.11.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 5.11.3
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 12.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 12.3.1
 - !ruby/object:Gem::Dependency
   name: redis-key_hash
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.50.0
+        version: 0.54.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.50.0
+        version: 0.54.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -133,6 +133,7 @@ files:
 - ".rubocop.yml"
 - ".travis.yml"
 - ".travis/nutcracker.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md