redis-ick 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 985bcc31664985af1016a12b0b45e015f6307b4c
-  data.tar.gz: f45617138cd637aeeafac5e6cc31fe3e103dc814
+  metadata.gz: 67d1648e28624b1ed5c0a9f475412deb102f2b5b
+  data.tar.gz: 97754295aead271f4748adef41f93eee3e081b11
 SHA512:
-  metadata.gz: ed52b22601623951ad500d91680910db9411829cf3792cbff8f435bd17536c2d5324285159f262b926f9d3be719b33d9293ed675f6ee10219a4d522d731356be
-  data.tar.gz: 6950898f2e07b6e678b1add4238cb66264c1f11740707fcd1dc0065b76d9e89a758c7c1eab36c2c31f7ae3dd5cf7ff2d8c2a37596136433be8e6b3ab8cf13575
+  metadata.gz: 020f83530d8dc2579d4343d32fdceec98dc1575f9137d2e0678ec0484a36fbfad14c3222cc1bddffa3548e8aa20bcffb0dd25b6ffc2b47c90bf76428c7405e70
+  data.tar.gz: e000d81edb08fefa4204c167d0adb3aae8f773892222681de2498e947e41a1af57370c83d3c29e2b5be504b3d2541c69836c8f1dc41b43c5d2301dabd1eb3b81

data/.rubocop.yml

@@ -1,6 +1,147 @@
-inherit_from: .rubocop_todo.yml
+
 AllCops:
   Include:
     - Rakefile
     - Gemfile
     - '*.gemspec'
+
+# Broadly speaking, test code gets a pass for most of the Metrics family.
+#
+# IMO test code is not the place get pedantic about class length,
+# method complexity, etc.  One should be encouraged to add more tests
+# with minimal friction, not forced to make a hard choice between
+# cutting tests or splitting up my test suites.
+#
+Metrics/ClassLength:
+  Max: 400
+  Exclude:
+    - 'test/**/*.rb'
+
+# I like this Metric in principle, but I don't like the default max of
+# 15.
+#
+# Also, as per Metrics/ClassLength IMO this kind of limit should not
+# apply to test code (I get up to 318 over there).
+#
+Metrics/AbcSize:
+  Max: 30
+  Exclude:
+    - 'test/**/*.rb'
+
+# I like this Metric in principle, but I don't like the default max of
+# 10.
+#
+# Also, as per Metrics/ClassLength IMO this kind of limit should not
+# apply to test code.
+#
+Metrics/MethodLength:
+  Max: 50
+  Exclude:
+    - 'test/**/*.rb'
+
+# I put extra spaces in a lot of expressions for a lot of different
+# reasons, including especially readability.
+#
+# I reject these cops.
+#
+Layout:
+  Enabled: false
+
+# I like a lot of the Lint tests, but not these.
+#
+Lint/AmbiguousBlockAssociation:           # obnoxiously rejects idiomatic Ruby
+  Enabled: false
+
+# Ick#ickreserve returns an Array of Arrays.
+#
+# Performance/HashEachMethods complains that:
+#
+#   ick.ickreserve('key',num).each do |message,score|
+#     ...
+#   end
+#
+# would be more efficient as a each_key loop.
+#
+# This cop inappropriately requires us to change working code to
+# broken code.  This is a known issue which the authors of Rubocop do
+# not intend to change:
+#
+#   https://github.com/bbatsov/rubocop/issues/4732
+#
+# My reaction is to blacklist this cop.
+#
+Performance/HashEachMethods:
+  Enabled: false
+
+# This does no more than insist I type "format" instead of "sprintf",
+# where the two are aliases.
+#
+Style/FormatString:
+  Enabled: false
+
+# There is nothing wrong with Ruby 1.9 Hash syntax.
+#
+Style/HashSyntax:
+  Enabled: false
+
+# No.  Indeed, postfix if can often drive a line over 80 columns wide.
+#
+Style/IfUnlessModifier:
+  Enabled: false
+
+# No.  There is nothing wrong with "if !foo".
+#
+# As far as I'm concerned, "unless" is in poor taste because it means
+# I have to think in English in two different logical senses - and
+# English is a poor language for logical senses.
+#
+Style/NegatedIf:
+  Enabled: false
+
+# "Do not use semicolons to terminate expressions."
+#
+# That's great when I terminate a single-line expression with a
+# redundant semicolo because I forget I'm not using C.
+#
+# But when I'm using a semicolon to separate two expressions there is
+# no other choice.  So this really ought to be Style/OneExprPerLine,
+# which I reject.
+#
+Style/Semicolon:
+  Enabled: false
+
+# No.
+#
+# Some lines must have '\"'.  It is ugly to use a mix of '"' and '\''
+# in LoCs which are close to one another.  Therefore, banning '"' if
+# '"' is not strictly necessary drives visual inconsistency.
+#
+Style/StringLiterals:
+  Enabled: false
+
+# This is the same kind of obnoxious pedantry which drove Hungarian
+# Notation.
+#
+# The [] literal syntax is perfectly servicable and there is no point
+# _tightly_ coupling it to the content of the array.  That's why we
+# have context-free grammers!
+#
+Style/SymbolArray:
+  Enabled: false
+
+# Shockingly, this cop requires us to *OMIT*, not *INCLUDE* parens in
+# ternery conditons.
+#
+# IMO this one is actively harmful in that it discourages attention to
+# clarity and avoiding some nasty precedence surprises.
+#
+Style/TernaryParentheses:
+  Enabled: false
+
+# I am a huge fan of using trailing commas when I break an argument
+# list down one-per line.
+#
+# As such, I reject this test.
+#
+Style/TrailingCommaInLiteral:
+  Enabled: false

data/.travis.yml

@@ -8,5 +8,7 @@ services:
 rvm:
   - 2.1.6
 script:
-  - rubocop
+  - rubocop --version
+  - bundle exec rubocop --version
+  - bundle exec rubocop
   - bundle exec env REDIS_URL=redis://localhost:6379 rake test

data/Gemfile

@@ -1,4 +1,4 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 ProsperWorks
+Copyright (c) 2015-2017 ProsperWorks
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,45 +1,264 @@
-# Redis::Ick ![TravisCI](https://travis-ci.org/ProsperWorks/redis-ick.svg?branch=master)
+# Redis::Ick An Indexing Queue ![TravisCI](https://travis-ci.org/ProsperWorks/redis-ick.svg?branch=master)
 
-Ick: An Indexing QUeue.
+Redis::Ick implements a priority queue in Redis which supports:
 
-Redis::Ick implements a priority queue in Redis with two-phase commit
-and write-folding aka write-combining semantics.
+* multiple producers
+* write-folding
+* two-phase commit for a single consumer
 
-Icks are well-suited for dirty lists in data sync systems.
+Ick solves a variety of race condition and starvation issues which can
+arise between the producers and the consumer.
 
-A Redis-based queue-like data structure for message passing in a
-many-producer/single-consumer pattern.
+Icks are similar to (and built from) Redis sorted sets.  They are
+well-suited for dirty lists in data sync systems.
 
-Ick is compatible with Redis Cluster and RedisLabs Enterprise Cluster.
-Each Ick has a master key and any other Redis keys it uses use a
-prescriptive hash based on the master key.
+Ick has been live in production at ProsperWorks since 2015-10-21.  We
+use them at the heart of our PG-to-ES and PG-to-Neo4j pipelines, for
+data migration and repair, and a variety of other crawler systems.
 
-Ick offers write-folding semantics in which re-adding a member already
-in queue does not increase the size of the queue.  It may, or may not,
-rearrange that member's position within the queue.
+## Background: The Original Pattern
 
-Ick is batchy on the consumer side with reliable delivery semantics
-using a two-phase protocol: it supports for reserving batches and
-later committing all, some, or none of them.
+Long before Ick, our indexer queue was a simple Redis sorted set which
+used current time for scores.  It looked like:
 
-Note that members held in the reserve buffer by the consumer do *not*
-write-fold against members being added by producers.
+    # in any process whenever a document is dirtied
+    redis.zadd(queue_key,Time.now.to_f,document_id)
 
-Ick offers atomicity among producer and consumer operations by virtue
-of leveraging Lua-in-Redis.
+    # in the indexer process
+    batch = redis.zrangebyrank(queue_key,0,batch_size)  # critical section start
+    process_batch_slowly(batch)
+    # burn down the queue only if the batch succeeded
+    redis.zrem(queue_key,*members_of(batch))            # critical section end
 
-Ick offers starvation-free semantics when scores are approximately the
-current time.  When Ick performs write-folding, it always preserves
-the *lowest* score seen for a given message.  Thus, in both the
-producer set and the consumer set, entries never move further away
-from the poppy end.
+**Big Advantage**: Failover.  Because we defer ZREM until after
+success, when we fail in process_batch_slowly() (such as via an
+exception or SIGKILL), all document_ids in the batch are still in
+Redis.  When the indexer process resumes, those document_ids will run
+again.
 
-Ick supports only a single consumer: there is only one buffer for the
-two-phase pop protocol.  If you need more than one consumer, shard
-messages across multiple Icks each of which routes to one consumer.
+**Big Advantage**: Write Folding.  Because we use Redis sorted sets,
+when a document is dirtied twice in quick succession, we only get 1
+entry in the queue.  We change the timestamp but we do not end up
+with 2 entries in the queue.  Thus, the queue grows only in the number
+of dirty _documents_ per unit time, not in the number of dirty
+_operations_ per unit time.  In a sense, the more we fall behind the
+slower we fall.
 
-incept: 2015-10-01
-arch:   https://goo.gl/V1g9I8
+**Big Problem**: The Forgotten Dirtiness Problem.  If some document is
+dirtied a second time after the start of process_batch_slowly(), when
+process_batch_slowly() end we will drop that document from the queue.
+Thus, the document will be dirty but no longer in the queue!
+
+**Small Problem**: The Hot Data Starvation Problem.  Because we score
+by time-of-dirtiness and we use ZRANGEBYRANK starting at 0, each batch
+is the _coldest_ dirty documents.  Most of the time this is a good
+proxy for what we really care about: the _oldest_ dirty documents.
+But when a document is re-dirtied, its old timestamp is replaced with
+a new timestamp.  In effect, it jumps from the cold end of the queue
+to the hot end of the queue.  If the queue is big enough that it is
+always larger than one batch and a document is hot enough that it gets
+updated in queue more often than our batches, the document will never
+be popped out into a batch.
+
+## Background: The Intermediate Pattern
+
+A year before Ick, August 2014, Gerald made a huge improvement which
+mostly mitigated the Forgotten Dirtiness Problem:
+
+    # in any process: whenever a document is dirtied
+    redis.zadd(queue_key,Time.now.to_f,document_id)
+
+    # in the indexer process:
+    batch1 = redis.zrangebyrank(queue_key,0,batch_size)
+    process_batch_slowly(batch)
+    # burn down the queue only if the batch succeeded
+    batch2 = redis.zrangebyrank(queue_key,0,batch_size) # critical section start
+    unchanged_keys = batch1.keys - keys_whose_score_changed_in(batch1,batch2)
+    redis.zrem(queue_key,*members_of(unchanged_keys))   # critical section end
+
+Gerald changed it so a second snapshot of the cold end of the queue is
+taken after process_batch_slowly().  Only documents whose timestamps
+did not change between the two snapshots are removed from the queue.
+
+Notice how the critical section no longer includes
+process_batch_slowly().  Instead it only spans two Redis ops and some
+local set arithmetic which.
+
+The critical section and the Forgotten Dirtiness Problem which it
+causes is still there, but is much smaller.  In practice we have
+process_batch_slowly() taking minutes, but even in extreme situations
+this critical section never took more than 3 seconds.
+
+## Proposal: The Ick Pattern
+
+In October 2015, while reviewing the Forgotten Dirtiness problem, we
+identified the Hot Data Starvation Problem.  We developed Ick and
+switched to this almost familiar pattern:
+
+    # in any process: whenever a document is dirtied
+    Ick.ickadd(redis,queue_key,Time.now.to_f,document_id)
+
+    # in the indexer process:
+    batch = Ick.ickreserve(redis,queue_key,batch_size)
+    process_batch_slowly(batch)
+    # burn down the queue only if the batch succeeded
+    Ick.ickcommit(redis,queue_key,*members_of(batch))   # critical section gone
+
+
+Ick solves for failover via a two phase commit protocol between
+**ickreserve** and **ickcommit**.  If there is a failure during
+process_batch_slowly(batch), the next time time we call **ickreserve**
+we will just get the same batch - it will have resided unchanged in
+the consumer set until we get happy and call **ickcommit**.
+
+Ick solves the Forgotten Dirtiness Problem by virtue of
+**ickreserve**’s implicit atomicity and the fact that **ickcommit** is
+only ever called from the indexer and producers do not mutate the
+consumer set.
+
+Ick solves the Hot Data Starvation Problem by a subtle change in
+ICKADD.  Unlike ZADD, which overwrites the old score when a message is
+re-added, or ZADD NX which always preserves the old score, ICKADD
+always takes the _min_ of the old and new scores.  Thus, Ick tracks
+the first-known ditry time for a message even when there is time skew
+in the producers.  The longer entries stay in the consumer set, the
+more they implicitly percolate toward the cold end regardless of how
+many updates they receive.  Ditto in the consumer set.  Provided that
+all producers make a best effort to use only current or future
+timestamps when they call ICKADD, the ICKRESERVE batch will always
+include the oldest entries and there will be no starvation.
+
+Apology: I know that [Two-Phase
+Commit](https://en.wikipedia.org/wiki/Two-phase_commit_protocol) has a
+different technical meaning than what Ick does.  Unfortunately I can't
+find a better name for this very common failsafe queue pattern.  I
+suppose we could think of the Redis sorted set as the coordinator and
+the consumer process as the (single) participant node and, generously,
+Two-Phase Commit might be taken to describe Ick.
+
+
+## What is Ick?
+
+An Ick is a collection of three Redis keys which all live on the same
+[Redis hash slot](https://redis.io/topics/cluster-spec):
+
+* version flag, a string
+* producer set, a sorted set into which we flag keys as dirty with timestamps
+* consumer set, a sorted set from which the indexer pulls batches to index
+
+### Ick defines 5 operations on this data via Lua on Redis:
+
+* **ickdel**: removes all keys associated with a given Ick structure
+* **ickstats**: returns a hash of stats including version and size
+* **ickadd**: add a batch of members with scores to the producer set
+  * implements write-folding: a message can only appear once in the producer set
+  * when a member is re-added, it takes the lowest of 2 scores
+* **ickreserve**: moves members from the producer set to the consumer
+  set until the consumer set is size N or the producer set is empty
+  * implements write-folding: a message can only appear once in the consumer set
+  * when a member-is re-added it takes the lowest of 2 scores
+  * returns the results as an array
+* **ickcommit**: deletes members from the consumer set
+
+Reminder: With few exceptions, all Redis commands are atomic and
+transactional.  This includes any Lua scripts such as those which
+implement Ick.  This atomicity guarantee is important to the
+correctness of Ick, but because it is inherent in Redis/Lua, does not
+appear explicitly in any of the Ick sources.
+
+## Fabulous Diagram
+
+Here is a coarse dataflow for members moving through an Ick.
+
+    app
+    |
+    +-- **ickadd** --> producer set
+                   |
+                   +-- **ickreserve** --> consumer set
+                                      |
+                                      +-- **ickcommit** --> forgotten
+
+## Miscellanea
+
+### Ready for Redis Cluster
+
+Even though one Ick uses three Redis keys, Ick is compatible with
+Redis Cluster.  At ProsperWorks we use it with RedisLabs Enterprise
+Cluster.
+
+Ick does some very tricky things to compute the producer set and
+consumer set keys from the master key in a way which puts them all on
+the same slot in both Redis Cluster and with RLEC's default
+prescriptive hashing algorithm.
+
+See [redis-key_hash](https://github.com/ProsperWorks/redis-key_hash)
+for how test this.
+
+### Scalability
+
+Ick supports only a single consumer: there is only one consumer set.
+
+If your application need more than one consumer for throughput or
+other reasons, you should shard across multiple Icks, each with one
+consumer loop each.
+
+This is exactly how we use Icks at ProsperWorks.  Our application code
+does not simply push to an individual Ick.  We push to a bit of code
+which knows that one "channel" is really N Icks.  To select an Ick,
+that code does a stable hash of our document_ids, modulo N.
+
+This way, each Ick is able to dedupe across only its dedicated subset
+of all messages.
+
+We considered a more complicated Ick which supported multiple
+consumers, but a lot of new problems come up once we take that step:
+can one message be in multiple consumer sets?  If not, what happens
+when one consumer halts?  How do we prevent the cold end of the
+producer set from getting clogged up with messages destined for the
+idle consumer?
+
+We prefer handling those issues in higher-level code.  Thus, Ick by
+itself does not attempt to solve scalability.
+
+
+### Some Surprises Which Can Be Gotchas in Test
+
+Because ICKADD uses write-folding semantics over the producer set,
+ICKADD might or might not grow the total size of the queue.
+
+ICKRESERVE is not a read-only operation.  It can mutate both the
+producer set and the consumer set.  Because ICKRESERVE uses
+write-folding semantics between the producer set and the consumer set,
+ICKRESERVE(N) might:
+
+* shrink the producer set by N and grow the consumer set by N
+* shrink the producer set by 0 and grow the consumer set by 0
+* shrink the producer set by N and grow the consumer set by 0
+* or anything in between
+
+Because Ick always uses the min when multiple scores are present for
+one message, ICKADD can rearrange the order of the producer set and
+ICKRESERVE can rearrange the order of the consumer set in surprising
+ways.
+
+ICKADD write-folds in the producer set but not in the consumer set.
+Thus, one message can appear in both the producer set and the consumer
+set.  At first this seems wrong and inefficient, but in fact it is a
+desirable property.  When a message is in both sets, it means it was
+included in a batch by ICKRESERVE, then added by ICKADD, but has yet
+to be ICKCOMMITed.  The interpretation for this is that the consumer
+is actively engaged in updating the downstream systems.  But that
+means, at the Ick it is indeterminate whether the message is still
+dirty or has been cleaned.  That is, being in both queues corresponds
+exactly to a message being in the critical section where a race
+condition is possible.  Thus, we _want_ it to still be dirty and to
+appear in a future batch.
+
+None of these surprises is a bug in Ick: they are all consistent with
+the design and the intent.  But they are surprises nonetheless and can
+(and have) led to bugs in code which makes an unwarranted assumption.
+
+## Installation
 
 ```ruby
 gem 'redis-ick'
@@ -67,6 +286,7 @@ Usage example for producers:
     ick = Ick.new(redis)
     ick.ickadd("mykey",12.8,"foo")
     ick.ickadd("mykey",123.4,"baz")
+    ick.ickadd("mykey",Time.now.to_f,"bang")   # Time.now recommended for scores
 
 Usage example for consumer:
 

data/Rakefile

@@ -1,10 +1,10 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/**/*_test.rb"]
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
 end
 
 task :default => :test

data/bin/console

@@ -1,14 +1,14 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "redis/ick"
+require 'bundler/setup'
+require 'redis/ick'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
+# require 'pry'
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/lib/redis/ick.rb

@@ -2,9 +2,11 @@ require 'redis/ick/version'
 require 'redis/script_manager'
 
 class Redis
+
+  # Binds Lua code to provide the Ick operations in Redis.
+  #
   class Ick
 
-    # TODO: rubocop
     # TODO: rdoc
 
     # Creates an Ick accessor.
@@ -18,14 +20,16 @@ class Redis
       if !redis.is_a?(Redis)
         raise ArgumentError, "not a Redis: #{redis}"
       end
-      if statsd && !statsd.respond_to?(:increment)
-        raise ArgumentError, "no statsd.increment"
-      end
-      if statsd && !statsd.respond_to?(:timing)
-        raise ArgumentError, "no statsd.timeing"
-      end
-      if statsd && !statsd.respond_to?(:time)
-        raise ArgumentError, "no statsd.time"
+      if statsd
+        if !statsd.respond_to?(:increment)
+          raise ArgumentError, 'no statsd.increment'
+        end
+        if !statsd.respond_to?(:timing)
+          raise ArgumentError, 'no statsd.timeing'
+        end
+        if !statsd.respond_to?(:time)
+          raise ArgumentError, 'no statsd.time'
+        end
       end
       @redis  = redis
       @statsd = statsd
@@ -61,10 +65,10 @@ class Redis
     def _statsd_time(metric)
       if statsd
         statsd.time(metric) do
-          return block_given? ? yield : nil
+          block_given? ? yield : nil
         end
       else
-        return block_given? ? yield : nil
+        block_given? ? yield : nil
       end
     end
 
@@ -258,16 +262,16 @@ class Redis
         class << raw_ickreserve_results
           alias_method :original_value, :value
           def value
-            original_value.each_slice(2).map { |p|
+            original_value.each_slice(2).map do |p|
               [ p[0], ::Redis::Ick._floatify(p[1]) ]
-            }
+            end
           end
         end
         raw_ickreserve_results
       else
-        results = raw_ickreserve_results.each_slice(2).map { |p|
+        results = raw_ickreserve_results.each_slice(2).map do |p|
           [ p[0], ::Redis::Ick._floatify(p[1]) ]
-        }
+        end
         _statsd_timing('profile.ick.ickreserve.num_results',results.size)
         results
       end
@@ -472,6 +476,7 @@ class Redis
       local ick_cset_size = redis.call('ZCARD',ick_cset_key)
       local ick_stats     = {
         'key',        ick_key,
+        'keys',       { ick_key, ick_pset_key, ick_cset_key },
         'ver',        ick_ver,
         'cset_size',  ick_cset_size,
         'pset_size',  ick_pset_size,
@@ -588,7 +593,8 @@ class Redis
         if ick_cset_size and target_cset_size <= ick_cset_size then
           break
         end
-        local first_in_pset  = redis.call('ZRANGE',ick_pset_key,0,0,'WITHSCORES')
+        local first_in_pset  = 
+          redis.call('ZRANGE',ick_pset_key,0,0,'WITHSCORES')
         if 0 == table.getn(first_in_pset) then
           break
         end

data/lib/redis/ick/version.rb

@@ -7,15 +7,28 @@ class Redis
     #
     # 0.0.2 - Broke out into Prosperworks/redis-ick, make public.
     #
-    # 0.0.3 - Got .travis.yml working with a live redis-server, some
-    #         cleanup, runtime dependency on redis-script_manager for
-    #         Ick._eval, added Rubocop.
+    # 0.0.3 - Got .travis.yml working with a live redis-server.
+    #
+    #         Runtime dependency on redis-script_manager for
+    #         Ick._eval.
+    #
+    #         Initial Rubocop integration.
+    #
+    #         Misc cleanup.
+    #
+    # 0.0.4 - Imported text from original design doc to README.md, polish.
+    #
+    #         Rubocop polish and defiance.
+    #
+    #         Development dependency on redis-key_hash to test
+    #         prescriptive hash claims.  Identified limits of
+    #         prescriptive hash robustness.
     #
     # 0.1.0 - (future) Big README.md and Rdoc update, solicit feedback
     #         from select external beta users.
     #
     # 0.2.0 - (future) Incorporate feedback, announce.
     #
-    VERSION = '0.0.3'
+    VERSION = '0.0.4'.freeze
   end
 end

data/redis-ick.gemspec

@@ -1,16 +1,15 @@
-# coding: utf-8
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "redis/ick/version"
+require 'redis/ick/version'
 
 Gem::Specification.new do |spec|
 
-  spec.name          = "redis-ick"
+  spec.name          = 'redis-ick'
   spec.version       = Redis::Ick::VERSION
   spec.platform      = Gem::Platform::RUBY
 
-  spec.authors       = ["jhwillett"]
-  spec.email         = ["jhw@prosperworks.com"]
+  spec.authors       = ['jhwillett']
+  spec.email         = ['jhw@prosperworks.com']
 
   spec.summary       = 'Redis queues with two-phase commit and write-folding.'
   spec.homepage      = 'https://github.com/ProsperWorks/redis-ick'
@@ -18,15 +17,18 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler',              '~> 1.14'
   spec.add_development_dependency 'rake',                 '~> 10.0'
   spec.add_development_dependency 'minitest',             '~> 5.0'
-  spec.add_development_dependency 'redis',                '~> 3.2'
+  spec.add_development_dependency 'redis-key_hash',       '~> 0.0.4'
+  spec.add_development_dependency 'redis-namespace',      '~> 1.5'
+  spec.add_development_dependency 'rubocop',              '~> 0.50.0'
 
+  spec.add_dependency             'redis',                '~> 3.2'
   spec.add_runtime_dependency     'redis-script_manager', '~> 0.0.2'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-ick
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - jhwillett
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: redis-key_hash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.50.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.50.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -59,7 +101,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -89,7 +131,6 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rubocop.yml"
-- ".rubocop_todo.yml"
 - ".travis.yml"
 - Gemfile
 - LICENSE

data/.rubocop_todo.yml

@@ -1,296 +0,0 @@
-# This configuration was generated by
-# `rubocop --auto-gen-config`
-# on 2017-08-29 15:15:59 -0700 using RuboCop version 0.49.1.
-# The point is for the user to remove these configuration records
-# one by one as the offenses are removed from the code base.
-# Note that changes in the inspected code, or installation of new
-# versions of RuboCop, may require this file to be generated again.
-
-# Offense count: 1
-# Cop supports --auto-correct.
-Layout/EmptyLineAfterMagicComment:
-  Exclude:
-    - 'redis-ick.gemspec'
-
-# Offense count: 6
-# Cop supports --auto-correct.
-# Configuration parameters: AllowAdjacentOneLineDefs, NumberOfEmptyLines.
-Layout/EmptyLineBetweenDefs:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-Layout/EmptyLines:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: empty_lines, no_empty_lines
-Layout/EmptyLinesAroundBlockBody:
-  Exclude:
-    - 'redis-ick.gemspec'
-
-# Offense count: 4
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines
-Layout/EmptyLinesAroundClassBody:
-  Exclude:
-    - 'lib/redis/ick.rb'
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 15
-# Cop supports --auto-correct.
-# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
-Layout/ExtraSpacing:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-Layout/LeadingCommentSpace:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 276
-# Cop supports --auto-correct.
-Layout/SpaceAfterComma:
-  Exclude:
-    - 'lib/redis/ick.rb'
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-Layout/SpaceAfterNot:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyleInsidePipes, SupportedStylesInsidePipes.
-# SupportedStylesInsidePipes: space, no_space
-Layout/SpaceAroundBlockParameters:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: SupportedStyles.
-# SupportedStyles: space, no_space
-Layout/SpaceAroundEqualsInParameterDefault:
-  EnforcedStyle: no_space
-
-# Offense count: 5
-# Cop supports --auto-correct.
-# Configuration parameters: AllowForAlignment.
-Layout/SpaceAroundOperators:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: space, no_space
-Layout/SpaceBeforeBlockBraces:
-  Exclude:
-    - 'lib/redis/ick.rb'
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 4
-# Cop supports --auto-correct.
-# Configuration parameters: AllowForAlignment.
-Layout/SpaceBeforeFirstArg:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 4
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, EnforcedStyleForEmptyBraces, SupportedStylesForEmptyBraces, SpaceBeforeBlockParameters.
-# SupportedStyles: space, no_space
-# SupportedStylesForEmptyBraces: space, no_space
-Layout/SpaceInsideBlockBraces:
-  Exclude:
-    - 'Gemfile'
-    - 'lib/redis/ick.rb'
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 10
-# Cop supports --auto-correct.
-Layout/SpaceInsideBrackets:
-  Exclude:
-    - 'lib/redis/ick.rb'
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 1
-Lint/AmbiguousBlockAssociation:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 18
-Metrics/AbcSize:
-  Max: 318
-
-# Offense count: 2
-# Configuration parameters: CountComments.
-Metrics/ClassLength:
-  Max: 609
-
-# Offense count: 1
-Metrics/CyclomaticComplexity:
-  Max: 8
-
-# Offense count: 3
-# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
-# URISchemes: http, https
-Metrics/LineLength:
-  Max: 81
-
-# Offense count: 19
-# Configuration parameters: CountComments.
-Metrics/MethodLength:
-  Max: 175
-
-# Offense count: 1
-Metrics/PerceivedComplexity:
-  Max: 8
-
-# Offense count: 1
-# Cop supports --auto-correct.
-Performance/TimesMap:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 2
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, ProceduralMethods, FunctionalMethods, IgnoredMethods.
-# SupportedStyles: line_count_based, semantic, braces_for_chaining
-# ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object
-# FunctionalMethods: let, let!, subject, watch
-# IgnoredMethods: lambda, proc, it
-Style/BlockDelimiters:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 1
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: nested, compact
-Style/ClassAndModuleChildren:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 2
-Style/ClassVars:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 1
-Style/Documentation:
-  Exclude:
-    - 'spec/**/*'
-    - 'test/**/*'
-    - 'lib/redis/ick.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: format, sprintf, percent
-Style/FormatString:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 1
-# Configuration parameters: MinBodyLength.
-Style/GuardClause:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 3
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
-# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
-Style/HashSyntax:
-  Exclude:
-    - 'Rakefile'
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 3
-# Cop supports --auto-correct.
-# Configuration parameters: MaxLineLength.
-Style/IfUnlessModifier:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 6
-# Configuration parameters: SupportedStyles.
-# SupportedStyles: snake_case, camelCase
-Style/MethodName:
-  EnforcedStyle: snake_case
-
-# Offense count: 1
-# Cop supports --auto-correct.
-Style/MutableConstant:
-  Exclude:
-    - 'lib/redis/ick/version.rb'
-
-# Offense count: 15
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: both, prefix, postfix
-Style/NegatedIf:
-  Exclude:
-    - 'lib/redis/ick.rb'
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: AllowMultipleReturnValues.
-Style/RedundantReturn:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: AllowAsExpressionSeparator.
-Style/Semicolon:
-  Exclude:
-    - 'test/redis/ick_test.rb'
-
-# Offense count: 28
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, ConsistentQuotesInMultiline.
-# SupportedStyles: single_quotes, double_quotes
-Style/StringLiterals:
-  Exclude:
-    - 'Gemfile'
-    - 'Rakefile'
-    - 'bin/console'
-    - 'lib/redis/ick.rb'
-    - 'redis-ick.gemspec'
-    - 'test/redis/ick_test.rb'
-    - 'test/test_helper.rb'
-
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: MinSize, SupportedStyles.
-# SupportedStyles: percent, brackets
-Style/SymbolArray:
-  EnforcedStyle: brackets
-
-# Offense count: 1
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, AllowSafeAssignment.
-# SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
-Style/TernaryParentheses:
-  Exclude:
-    - 'lib/redis/ick.rb'
-
-# Offense count: 11
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyleForMultiline, SupportedStylesForMultiline.
-# SupportedStylesForMultiline: comma, consistent_comma, no_comma
-Style/TrailingCommaInLiteral:
-  Exclude:
-    - 'test/redis/ick_test.rb'