redis-key_hash 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c7dd84fcafb7fdfa166839a664d177a851ad7bf5
+  data.tar.gz: 496ced31fb42c55927fc739c6e5a930f6ea046f2
+SHA512:
+  metadata.gz: 9aa7190e2b7e7f50651d8628a1b7488e864f4a11cb803f25553f2233480f37f97af38024a335502dc93e3a7c7b3d2d451676b50399b81e40ff1d75e210a0487c
+  data.tar.gz: 7356c4f3ecb5efe65b4c8e8fff4bc67d70da7cc4560974d06736dad5f1a171a9bbb80307dfc7f373dee23af6cbc9e8daace89ad53c5d2fa7890ea260756f05c7

data/.gitignore

@@ -0,0 +1,52 @@
+*.gem
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# For a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+#
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.1.6
+before_install: gem install bundler -v 1.14.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in redis-key_hash.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 ProsperWorks
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# Redis::KeyHash
+
+Tests of key hash slot agreement for use with Redis Cluster and
+RedisLabs Enterprise Cluster.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'redis-key_hash'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install redis-key_hash
+
+## Usage
+
+    require 'redis/key_hash'
+    
+    # As syntactic sugar, expose Redis::KeyHash methods in the Redis class.
+    #
+    Redis.include Redis::KeyHash
+    
+    # Test whether several keys will hash to the same slot.
+    #
+    if Redis.all_in_one_slot?('a','b')
+      'happy'
+    else
+      'sad'
+    end
+    Redis.all_in_one_slot!('a','b') # may raise Redis::ImpendingCrossSlotError
+
+## Development
+
+After checking out the repo, run `bin/setup` to install
+dependencies. Then, run `rake test` to run the tests. You can also run
+`bin/console` for an interactive prompt that will allow you to
+experiment.
+
+To install this gem onto your local machine, run `bundle exec rake
+install`. To release a new version, update the version number in
+`version.rb`, and then run `bundle exec rake release`, which will
+create a git tag for the version, push git commits and tags, and push
+the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/ProsperWorks/redis-key_hash.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "redis/key_hash_util"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/redis/impending_cross_slot_error.rb

@@ -0,0 +1,18 @@
+# Raised from Redis::KeyHash.all_in_one_slot! on a mismatch.
+#
+# Generally, this indicates we were about to risk a Redis operation
+# which is likely to produce a Redis error result like:
+#
+#  (error) CROSSSLOT Keys in request don't hash to the same slot
+#
+# TODO: rubocop
+#
+# TODO: rdoc
+#
+class Redis
+  class ImpendingCrossSlotError < ArgumentError # TODO: Redis::CommandError?
+    #
+    # TODO: capture the problems with the keys?
+    #
+  end
+end

data/lib/redis/key_hash.rb

@@ -0,0 +1,214 @@
+require 'redis/key_hash/version'
+require 'redis/impending_cross_slot_error'
+
+module Redis::KeyHash
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    # TODO: rubocop
+
+    # TODO: rdoc
+
+    #
+    # TODO: KNOWN_STYLES hash, etc, move comments around
+    #
+
+    # The default for all_in_one_slot? and all_in_one_slot! is
+    # [:rc,:rlec] to encourage the of portable persistent hashing
+    # schemes.
+    #
+    # A scheme which works with both side-steps a potential vendor
+    # lock-in.
+    #
+    DEFAULT_STYLES ||= [ :rc, :rlec ].freeze
+
+    # The default style for hash_tag and hash_slot is :rc because we
+    # only know the hashing algorithm for Redis Cluster.
+    #
+    # We can guess that the hash for RedisLabs Enterprise Cluster is
+    # similar, but it is not documented.
+    #
+    DEFAULT_STYLE  ||= :rc
+
+    # Tests whether all of keys will hash to the same slot in all
+    # specified sharding styles.
+    #
+    # @param keys String keys to be tested
+    #
+    # @param namespace String or nil if non-nil, applied as a prefix to
+    # all keys as per the redis-namespace gem before testing.
+    #
+    # @param styles Array of Symbols and/or Regexps as per hash_tag().
+    #
+    # @return true if all of keys provably have the same hash_slot under
+    # all styles by virtue of having a single hash_tag, false otherwise.
+    #
+    def all_in_one_slot?(*keys, namespace: nil, styles: DEFAULT_STYLES)
+      begin
+        all_in_one_slot!(*keys, namespace: namespace, styles: styles)
+      rescue Redis::ImpendingCrossSlotError
+        return false
+      else
+        return true
+      end
+    end
+
+    # Like all_in_one_slot?, mismatch raises Redis::ImpendingCrossSlotError.
+    #
+    # @param keys String keys to be tested
+    #
+    # @param namespace String or nil if non-nil, applied as a prefix to
+    # all keys as per the redis-namespace gem before testing.
+    #
+    # @param styles Array of Symbols and/or Regexps as per hash_tag().
+    #
+    # @return true if all of keys will hash to the same slot in all of
+    # the styles, false if there is any doubt.
+    #
+    # @return true if all of keys provably have the same hash_slot
+    # under all styles by virtue of having a single hash_tag.
+    #
+    # @raises Redis::ImpendingCrossSlotError if, for any styles, the
+    # keys have a different hash_tag hence will not provably have the
+    # same hash_slot
+    #
+    def all_in_one_slot!(*keys, namespace: nil, styles: DEFAULT_STYLES)
+      nkeys       = namespace ? keys.map { |key| "#{namespace}:#{key}" } : keys
+      style2slot  = Hash.new
+      problems    = []
+      styles.each do |style|
+        tags      = nkeys.map { |nkey| hash_tag(nkey,style: style) }.uniq
+        next if tags.size <= 1
+        problems << "style #{style} sees tags #{tags.join(',')}"
+      end
+      if 0 != problems.size
+        err  = "CROSSSLOT"
+        err += " namespace=#{namespace}"
+        err += " keys=#{keys}"
+        err += " problems=#{problems}"
+        raise Redis::ImpendingCrossSlotError, err
+      end
+      true
+    end
+
+    # Computes the hash tag for a given key under a given Redis
+    # clustering algorithm.
+    #
+    # The :style => :rc implementation matches Redis Cluster exactly
+    # and is taken largely from the reference example provided in
+    # http://redis.io/topics/cluster-spec.
+    #
+    # The :style => :rlec implementation is partly speculative.  It is
+    # mostly interpreted from the default RedisLabs Enterprise Cluster
+    # document
+    # https://redislabs.com/redis-enterprise-documentation/concepts-architecture/architecture/database-clustering/
+    # plus our experience at ProsperWorks that, out of the box, RLEC
+    # uses the *last* {}-expr, not the first as in RC, from :rc (which
+    # we would not care about), they are also structually different
+    # (which can cause bugs unless we take special care).  :rc uses
+    # the first {}-expr in the, :rlec uses the last {}-expr.
+    #
+    # @param String key to be hashed
+    #
+    # @param Symbol :rc or rlec or Regexp which defines one capture group
+    #
+    # @param String the tag extracted from key as appropriate for :style.
+    #
+    def hash_tag(key, style: DEFAULT_STYLE)
+      regexp = nil
+      case style
+      when :rc
+        regexp = /{([^}]*)}/     # RC uses the first {}-expr
+      when :rlec
+        regexp = /.*\{(.*)\}.*/  # RLEC default per docs, uses last {}-expr
+      when Regexp
+        regexp = style           # you can define your own
+      end
+      if !regexp
+        raise ArgumentError, "bogus style #{style}"
+      end
+      match = regexp.match(key)
+      return match ? match[1] : key
+    end
+
+    # Computes the Redis hash_slot for a given key.
+    #
+    # Uses :style as per hash_tag, but performs hashing as per RC
+    # only.  We know through documentation and experimentation that RC
+    # uses crc16() and modulo 16384.  We do not know what RLEC does,
+    # but until we have a better model we assume it is the same.  This
+    # is probably a false assumption since the RLEC docs state that
+    # the number of shards can vary from cluster to cluster.  But for
+    # many analyses, using the same hash as RC is still useful.
+    #
+    # @param String key to be hashed
+    #
+    # @param Symbol :rc or :rlec or Regexp which defines one capture group
+    #
+    # @param non-negative Integer the hash which Redis will use to slot key
+    #
+    def hash_slot(key, style: DEFAULT_STYLE)
+      tag = hash_tag(key, style: style)
+      crc16(tag) % 16384
+    end
+
+    # Computes the Redis crc16 for a given key, as per the reference
+    # implementation provided in http://redis.io/topics/cluster-spec.
+    #
+    # This implementation is taken largely from that reference document,
+    # changed only slightly to port to Ruby.
+    #
+    # @param String key
+    #
+    # @param non-negative Integer the crc16 which Redis will use to
+    # compute a hash_key.
+    #
+    def crc16(key)
+      crc = 0
+      key.each_char do |char|
+        crc = ((crc << 8) & 0xFFFF) ^ CRC16TAB[((crc >> 8) ^ char.ord) & 0x00FF]
+      end
+      crc
+    end
+
+    CRC16TAB ||= [
+      0x0000,0x1021,0x2042,0x3063,0x4084,0x50a5,0x60c6,0x70e7,
+      0x8108,0x9129,0xa14a,0xb16b,0xc18c,0xd1ad,0xe1ce,0xf1ef,
+      0x1231,0x0210,0x3273,0x2252,0x52b5,0x4294,0x72f7,0x62d6,
+      0x9339,0x8318,0xb37b,0xa35a,0xd3bd,0xc39c,0xf3ff,0xe3de,
+      0x2462,0x3443,0x0420,0x1401,0x64e6,0x74c7,0x44a4,0x5485,
+      0xa56a,0xb54b,0x8528,0x9509,0xe5ee,0xf5cf,0xc5ac,0xd58d,
+      0x3653,0x2672,0x1611,0x0630,0x76d7,0x66f6,0x5695,0x46b4,
+      0xb75b,0xa77a,0x9719,0x8738,0xf7df,0xe7fe,0xd79d,0xc7bc,
+      0x48c4,0x58e5,0x6886,0x78a7,0x0840,0x1861,0x2802,0x3823,
+      0xc9cc,0xd9ed,0xe98e,0xf9af,0x8948,0x9969,0xa90a,0xb92b,
+      0x5af5,0x4ad4,0x7ab7,0x6a96,0x1a71,0x0a50,0x3a33,0x2a12,
+      0xdbfd,0xcbdc,0xfbbf,0xeb9e,0x9b79,0x8b58,0xbb3b,0xab1a,
+      0x6ca6,0x7c87,0x4ce4,0x5cc5,0x2c22,0x3c03,0x0c60,0x1c41,
+      0xedae,0xfd8f,0xcdec,0xddcd,0xad2a,0xbd0b,0x8d68,0x9d49,
+      0x7e97,0x6eb6,0x5ed5,0x4ef4,0x3e13,0x2e32,0x1e51,0x0e70,
+      0xff9f,0xefbe,0xdfdd,0xcffc,0xbf1b,0xaf3a,0x9f59,0x8f78,
+      0x9188,0x81a9,0xb1ca,0xa1eb,0xd10c,0xc12d,0xf14e,0xe16f,
+      0x1080,0x00a1,0x30c2,0x20e3,0x5004,0x4025,0x7046,0x6067,
+      0x83b9,0x9398,0xa3fb,0xb3da,0xc33d,0xd31c,0xe37f,0xf35e,
+      0x02b1,0x1290,0x22f3,0x32d2,0x4235,0x5214,0x6277,0x7256,
+      0xb5ea,0xa5cb,0x95a8,0x8589,0xf56e,0xe54f,0xd52c,0xc50d,
+      0x34e2,0x24c3,0x14a0,0x0481,0x7466,0x6447,0x5424,0x4405,
+      0xa7db,0xb7fa,0x8799,0x97b8,0xe75f,0xf77e,0xc71d,0xd73c,
+      0x26d3,0x36f2,0x0691,0x16b0,0x6657,0x7676,0x4615,0x5634,
+      0xd94c,0xc96d,0xf90e,0xe92f,0x99c8,0x89e9,0xb98a,0xa9ab,
+      0x5844,0x4865,0x7806,0x6827,0x18c0,0x08e1,0x3882,0x28a3,
+      0xcb7d,0xdb5c,0xeb3f,0xfb1e,0x8bf9,0x9bd8,0xabbb,0xbb9a,
+      0x4a75,0x5a54,0x6a37,0x7a16,0x0af1,0x1ad0,0x2ab3,0x3a92,
+      0xfd2e,0xed0f,0xdd6c,0xcd4d,0xbdaa,0xad8b,0x9de8,0x8dc9,
+      0x7c26,0x6c07,0x5c64,0x4c45,0x3ca2,0x2c83,0x1ce0,0x0cc1,
+      0xef1f,0xff3e,0xcf5d,0xdf7c,0xaf9b,0xbfba,0x8fd9,0x9ff8,
+      0x6e17,0x7e36,0x4e55,0x5e74,0x2e93,0x3eb2,0x0ed1,0x1ef0
+    ].freeze
+
+  end
+
+end

data/lib/redis/key_hash/version.rb

@@ -0,0 +1,17 @@
+class Redis
+  module KeyHash
+    #
+    # Version plan:
+    #
+    # 0.0.1 - still in Prosperworks/ALI/vendor/gems/redis-key_hash
+    #
+    # 0.0.2 - broke out into Prosperworks/redis-key_hash
+    #
+    # 0.1.0 - big README.md and Rdoc update, open repo
+    #
+    # 0.2.0 - solicit and incorporate initial feedback from select
+    #         beta external users
+    #
+    VERSION = "0.0.2" # broken into standalone repo, not polished
+  end
+end

data/redis-key_hash.gemspec

@@ -0,0 +1,57 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'redis/key_hash/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "redis-key_hash"
+  spec.version       = Redis::KeyHash::VERSION
+  spec.authors       = [
+    "jhwillett"
+  ]
+  spec.email         = [
+    "jhw@prosperworks.com"
+  ]
+  spec.license       = 'MIT'
+
+  spec.summary       = 'Tests Redis Cluster key hash slot agreement'
+  spec.homepage      = 'https://github.com/ProsperWorks/redis-key_hash'
+  spec.description   = %{
+redis-key_hash provides tests of key hash slot agreement for use with
+Redis Cluster and RedisLabs Enterprise Cluster.
+
+Redis Cluster (RC) and RedisLabs Enterprise Cluster (RLEC) both
+require that all keys in multi-key operations hash to the same node.
+They use slightly different default hashing algorithms and RLEC offers
+customizable hash pattersn.
+
+It is impossible to predict which nodes will host each shard, and
+difficult to predict which slot will host each key.  But it is easy to
+identify the key hash tag which RC or RLEC will use to select a shard.
+Where key hash tags agree, we can be certain that slots and nodes will
+also agree - even if hash tag disagreement does not always imply slot
+or node disagreement.
+
+By pulling this check into Ruby we can arrange to fail fast, before
+sending a command to Redis which is at risk of CROSSLOT failure.  We
+can also validate key management schemes in unit tests which run
+without support services or with a non-sharded Redis.
+}
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.require_paths = ["lib"]
+
+  # We use "foo: bar" syntax liberally, not the older ":foo => bar".
+  # Possibly other Ruby 2-isms as well.
+  #
+  spec.required_ruby_version = '~> 2.0'
+
+  spec.add_development_dependency "bundler",  "~> 1.14"
+  spec.add_development_dependency "rake",     "~> 10.5"
+  spec.add_development_dependency "minitest", "~> 5.10"
+
+  # This gem is pure Ruby, no weird dependencies.
+  #
+  spec.platform = Gem::Platform::RUBY
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: redis-key_hash
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- jhwillett
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+description: |2
+
+  redis-key_hash provides tests of key hash slot agreement for use with
+  Redis Cluster and RedisLabs Enterprise Cluster.
+
+  Redis Cluster (RC) and RedisLabs Enterprise Cluster (RLEC) both
+  require that all keys in multi-key operations hash to the same node.
+  They use slightly different default hashing algorithms and RLEC offers
+  customizable hash pattersn.
+
+  It is impossible to predict which nodes will host each shard, and
+  difficult to predict which slot will host each key.  But it is easy to
+  identify the key hash tag which RC or RLEC will use to select a shard.
+  Where key hash tags agree, we can be certain that slots and nodes will
+  also agree - even if hash tag disagreement does not always imply slot
+  or node disagreement.
+
+  By pulling this check into Ruby we can arrange to fail fast, before
+  sending a command to Redis which is at risk of CROSSLOT failure.  We
+  can also validate key management schemes in unit tests which run
+  without support services or with a non-sharded Redis.
+email:
+- jhw@prosperworks.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/redis/impending_cross_slot_error.rb
+- lib/redis/key_hash.rb
+- lib/redis/key_hash/version.rb
+- redis-key_hash.gemspec
+homepage: https://github.com/ProsperWorks/redis-key_hash
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Tests Redis Cluster key hash slot agreement
+test_files: []