clandestined 1.0.0a

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 47d9cefad2b6c0c74a35295e37f64aa81f8bca19
+  data.tar.gz: ce0099265bc7b8b3929228550a657edc1057e027
+SHA512:
+  metadata.gz: cd241d8e5896d6608f81477ea23ec575bc9f55e71ca3917705a4217cf7d84a22e5c916271413d6d42a4bd2e81a73af34c4b1bd9ae57ae72a10346dbb84804b26
+  data.tar.gz: f343ef64e349cc3805b1f8e994fc9bc95865e150fad80e15f75840511f90ff7e3250bf66d3e01da69594e413f236097f78b63a57f31242672a43c2c5c8c8cf9d

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle
+/bin
+/vendor
+/.vendor
+
+/pkg
+/tmp
+/lib/murmur3_native.bundle
+/lib/murmur3_native.so

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - jruby-18mode
+  - jruby-19mode

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kissmetrics_core_api-admin-client.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,21 @@
+PATH
+  remote: .
+  specs:
+    clandestined (1.0.0a)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (0.9.6)
+    rake-compiler (0.8.3)
+      rake
+    rubydoctest (1.1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  clandestined!
+  rake (= 0.9.6)
+  rake-compiler (= 0.8.3)
+  rubydoctest (= 1.1.3)

data/README.md

@@ -0,0 +1,147 @@
+clandestined
+===========
+
+rendezvous hashing implementation based on murmur3 hash
+
+
+## motiviation
+
+in distributed systems, the need often arises to locate objects amongst a
+cluster of machines. consistent hashing and rendezvous hashing are methods of
+performing this task, while minimizing data movement on cluster topology
+changes.
+
+clandestined is a library for rendezvous hashing which has the goal of simple
+clients and ease of use.
+
+Currently targetting for support:
+  - Ruby 1.8.6 through Ruby 2.1.x
+
+[![Build Status](https://travis-ci.org/ewdurbin/clandestined-ruby.svg?branch=master)](https://travis-ci.org/ewdurbin/clandestined-ruby)
+
+## example usage
+
+```ruby
+>> require 'clandestined/cluster'
+>>
+>> nodes = Hash[
+     '1' => Hash['name' => 'node1.example.com', 'zone' => 'us-east-1a'],
+     '2' => Hash['name' => 'node2.example.com', 'zone' => 'us-east-1a'],
+     '3' => Hash['name' => 'node3.example.com', 'zone' => 'us-east-1a'],
+     '4' => Hash['name' => 'node4.example.com', 'zone' => 'us-east-1b'],
+     '5' => Hash['name' => 'node5.example.com', 'zone' => 'us-east-1b'],
+     '6' => Hash['name' => 'node6.example.com', 'zone' => 'us-east-1b'],
+     '7' => Hash['name' => 'node7.example.com', 'zone' => 'us-east-1c'],
+     '8' => Hash['name' => 'node8.example.com', 'zone' => 'us-east-1c'],
+     '9' => Hash['name' => 'node9.example.com', 'zone' => 'us-east-1c'],
+     ]
+>>
+>> cluster = Cluster.new(nodes)
+>> cluster.find_nodes('mykey')
+=> ["4", "8"]
+```
+
+by default, `Cluster` will place 2 replicas around the cluster taking care to
+place the second replica in a separate zone from the first.
+
+in the event that your cluster doesn't need zone awareness, you can either
+invoke the `RendezvousHash` class directly, or use a `Cluster` with replicas
+set to 1
+
+```ruby
+>> require 'clandestined/cluster'
+>> require 'clandestined/rendezvous_hash'
+>>
+>> nodes = Hash[
+     '1' => Hash['name' => 'node1.example.com'],
+     '2' => Hash['name' => 'node2.example.com'],
+     '3' => Hash['name' => 'node3.example.com'],
+     '4' => Hash['name' => 'node4.example.com'],
+     '5' => Hash['name' => 'node5.example.com'],
+     '6' => Hash['name' => 'node6.example.com'],
+     '7' => Hash['name' => 'node7.example.com'],
+     '8' => Hash['name' => 'node8.example.com'],
+     '9' => Hash['name' => 'node9.example.com'],
+     ]
+>>
+>> cluster = Cluster.new(nodes, 1)
+>> rendezvous = RendezvousHash.new(nodes.keys)
+>>
+>> cluster.find_nodes('mykey')
+=> ["4"]
+>> rendezvous.find_node('mykey')
+=> "4"
+```
+
+## advanced usage
+
+### murmur3 seeding
+
+if you plan to use keys based on untrusted input (not really supported, but go
+ahead), it would be best to use a custom seed for hashing. although this
+technique is by no means a way to fully mitigate a DoS attack using crafted
+keys, it may make you sleep better at night.
+
+```ruby
+>> require 'clandestined/cluster'
+>> require 'clandestined/rendezvous_hash'
+>>
+>> nodes = Hash[
+     '1' => Hash['name' => 'node1.example.com'],
+     '2' => Hash['name' => 'node2.example.com'],
+     '3' => Hash['name' => 'node3.example.com'],
+     '4' => Hash['name' => 'node4.example.com'],
+     '5' => Hash['name' => 'node5.example.com'],
+     '6' => Hash['name' => 'node6.example.com'],
+     '7' => Hash['name' => 'node7.example.com'],
+     '8' => Hash['name' => 'node8.example.com'],
+     '9' => Hash['name' => 'node9.example.com'],
+     ]
+>>
+>> cluster = Cluster.new(nodes, 1, 1337)
+>> rendezvous = RendezvousHash.new(nodes.keys, 1337)
+>>
+>> cluster.find_nodes('mykey')
+=> ["7"]
+>> rendezvous.find_node('mykey')
+=> "7"
+```
+
+### supplying your own hash function
+
+a more robust, but possibly slower solution to mitigate DoS vulnerability by
+crafted key might be to supply your own cryptograpic hash function.
+
+in order for this to work, your method must be supplied to the `RendezvousHash`
+or `Cluster` object as a callable which takes a byte string `key` and returns
+an integer.
+
+```ruby
+>> require 'digest'
+>> require 'clandestined/cluster'
+>> require 'clandestined/rendezvous_hash'
+>>
+>> nodes = Hash[
+     '1' => Hash['name' => 'node1.example.com'],
+     '2' => Hash['name' => 'node2.example.com'],
+     '3' => Hash['name' => 'node3.example.com'],
+     '4' => Hash['name' => 'node4.example.com'],
+     '5' => Hash['name' => 'node5.example.com'],
+     '6' => Hash['name' => 'node6.example.com'],
+     '7' => Hash['name' => 'node7.example.com'],
+     '8' => Hash['name' => 'node8.example.com'],
+     '9' => Hash['name' => 'node9.example.com'],
+     ]
+>>
+>> def my_hash_function(key)
+     Digest::SHA1.hexdigest(key).to_i(16)
+     end
+>>
+>> cluster = Cluster.new(nodes, 1, 0, method(:my_hash_function))
+>> rendezvous = RendezvousHash.new(nodes.keys, 0, method(:my_hash_function))
+>>
+>> cluster.find_nodes('mykey')
+=> ["1"]
+>> rendezvous.find_node('mykey')
+=> "1"
+```

data/Rakefile

@@ -0,0 +1,33 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+require 'rake/extensiontask'
+
+def can_compile_extensions
+  if defined? RUBY_DESCRIPTION
+    if RUBY_DESCRIPTION =~ /jruby/
+      false
+    else
+      true
+    end
+  else
+    true
+  end
+end
+
+Rake::ExtensionTask.new('murmur3_native')
+
+Rake::TestTask.new do |t|
+  t.libs << "ext"
+  t.libs << "test"
+  t.test_files = FileList['test/test*.rb']
+end
+
+task :test_docs do
+  system("rubydoctest README.md") or exit!(1)
+end
+
+if can_compile_extensions
+  task :default => [:compile, :test, :test_docs]
+else
+  task :default => [:test, :test_docs]
+end

data/clandestined.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name = 'clandestined'
+  s.version = '1.0.0a'
+  s.date = Time.now.strftime('%Y-%m-%d')
+  s.summary = 'rendezvous hashing implementation based on murmur3 hash'
+  s.author = "Ernest W. Durbin III"
+  s.email = 'ewdurbin@gmail.com'
+  s.homepage = 'https://github.com/ewdurbin/clandestined-ruby'
+
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files -- test/*`.split("\n")
+  s.require_paths = ['lib', 'ext']
+  s.extensions = ['ext/murmur3_native/extconf.rb']
+
+  s.add_development_dependency 'rake', '0.9.6'
+  s.add_development_dependency 'rake-compiler', '0.8.3'
+  s.add_development_dependency 'rubydoctest', '1.1.3'
+
+  s.has_rdoc = false
+
+  s.description = <<DESCRIPTION
+rendezvous hashing implementation based on murmur3 hash
+DESCRIPTION
+end

data/ext/murmur3_native/extconf.rb

@@ -0,0 +1,22 @@
+can_compile_extensions = false
+
+begin
+  require 'mkmf'
+  can_compile_extensions = true
+  dir_config("murmur3_native")
+  have_library("c", "main")
+  $defs << "-DRUBY_VERSION_CODE=#{RUBY_VERSION.gsub(/\D/, '')}"
+rescue Exception
+  $stderr.puts "Could not require 'mkmf'. Not fatal, the extensions are optional."
+end
+
+
+if can_compile_extensions
+  create_makefile("murmur3_native")
+else
+  open("Makefile", "wb") do |mfile|
+    mfile.puts '.PHONY: install'
+    mfile.puts 'install:'
+    mfile.puts "\t" + '@echo "Extensions not installed, falling back to pure Ruby version."'
+  end
+end

data/ext/murmur3_native/murmur3_native.c

@@ -0,0 +1,77 @@
+#include "ruby.h"
+
+#include <stdint.h>
+#include <string.h>
+
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+uint32_t murmur3_32(const char *key, uint32_t len, uint32_t seed) {
+    static const uint32_t c1 = 0xcc9e2d51;
+    static const uint32_t c2 = 0x1b873593;
+    static const uint32_t r1 = 15;
+    static const uint32_t r2 = 13;
+    static const uint32_t m = 5;
+    static const uint32_t n = 0xe6546b64;
+ 
+    uint32_t hash = seed;
+ 
+    const int nblocks = len / 4;
+    const uint32_t *blocks = (const uint32_t *) key;
+    int i;
+    for (i = 0; i < nblocks; i++) {
+        uint32_t k = blocks[i];
+        k *= c1;
+        k = (k << r1) | (k >> (32 - r1));
+        k *= c2;
+ 
+        hash ^= k;
+        hash = ((hash << r2) | (hash >> (32 - r2))) * m + n;
+    }
+ 
+    const uint8_t *tail = (const uint8_t *) (key + nblocks * 4);
+    uint32_t k1 = 0;
+ 
+    switch (len & 3) {
+    case 3:
+        k1 ^= tail[2] << 16;
+    case 2:
+        k1 ^= tail[1] << 8;
+    case 1:
+        k1 ^= tail[0];
+ 
+        k1 *= c1;
+        k1 = (k1 << r1) | (k1 >> (32 - r1));
+        k1 *= c2;
+        hash ^= k1;
+    }
+ 
+    hash ^= len;
+    hash ^= (hash >> 16);
+    hash *= 0x85ebca6b;
+    hash ^= (hash >> 13);
+    hash *= 0xc2b2ae35;
+    hash ^= (hash >> 16);
+ 
+    return hash;
+}
+
+static VALUE rb_mumur3_32(int argc, VALUE* argv, VALUE self) {
+    VALUE rstr;
+    if (argc == 0 || argc > 2) {
+	rb_raise(rb_eArgError, "accept 1 or 2 arguments: (string[, seed])");
+    }
+    rstr = argv[0];
+    StringValue(rstr);
+    uint32_t value = murmur3_32(RSTRING_PTR(rstr), RSTRING_LEN(rstr), argc == 1 ? 0 : NUM2UINT(argv[1]));
+    return UINT2NUM(value);
+}
+
+VALUE Murmur3Native = Qnil;
+
+void Init_murmur3_native();
+
+void Init_murmur3_native() {
+    VALUE Murmur3Native = rb_define_module("Murmur3Native");
+    rb_define_module_function(Murmur3Native, "murmur3_32", rb_mumur3_32, -1);
+}

data/lib/clandestined/cluster.rb

@@ -0,0 +1,111 @@
+
+require 'clandestined/rendezvous_hash'
+require 'murmur3'
+
+class Cluster
+
+  include Murmur3
+
+  attr_reader :hash_function
+  attr_reader :murmur_seed
+  attr_reader :replicas
+  attr_reader :nodes
+  attr_reader :zones
+  attr_reader :zone_members
+  attr_reader :rings
+
+  def initialize(cluster_config=nil, replicas=2, murmur_seed=0, hash_function=method(:murmur3_32))
+    @murmur_seed = murmur_seed
+
+    if hash_function == method(:murmur3_32)
+      @hash_function = lambda { |key| hash_function.call(key, murmur_seed) }
+    elsif murmur_seed != 0
+      raise ArgumentError, 'Cannot apply seed to custom hash function #{hash_function}'
+    else
+      @hash_function = hash_function
+    end
+
+    @replicas = replicas
+    @nodes = Hash[]
+    @zones = []
+    @zone_members = Hash[]
+    @rings = Hash[]
+
+    if cluster_config
+      cluster_config.each do |node, node_data|
+        name = node_data['name']
+        zone = node_data['zone']
+        add_zone(zone)
+        add_node(node, zone, name)
+      end
+    end
+  end
+
+  def add_zone(zone)
+    @zones.push(zone) unless zones.include?(zone)
+    unless zone_members.has_key?(zone)
+      @zone_members[zone] = []
+    end
+    @zones.sort!
+  end
+
+  def remove_zone(zone)
+    if zones.include?(zone)
+      @zones.delete(zone)
+      for member in zone_members[zone]
+        @nodes.delete(member)
+      end
+      @zones.sort!
+      @rings.delete(zone)
+      @zone_members.delete(zone)
+    end
+  end
+
+  def add_node(node_id, node_zone=nil, node_name=nil)
+    if nodes.include?(node_id)
+        raise ArgumentError, 'Node with id #{node_id} already exists'
+    end
+    add_zone(node_zone)
+    unless rings.has_key?(node_zone)
+      @rings[node_zone] = RendezvousHash.new(nil, 0, self.hash_function)
+    end
+    @rings[node_zone].add_node(node_id)
+    @nodes[node_id] = node_name
+    unless zone_members.has_key?(node_zone)
+      @zone_members[node_zone] = []
+    end
+    @zone_members[node_zone].push(node_id)
+  end
+
+  def remove_node(node_id, node_zone=nil, node_name=nil)
+    @rings[node_zone].remove_node(node_id)
+    @nodes.delete(node_id)
+    @zone_members[node_zone].delete(node_id)
+    if zone_members[node_zone].length == 0
+        remove_zone(node_zone)
+    end
+  end
+
+  def node_name(node_id)
+    nodes[node_id]
+  end
+
+  def find_nodes(search_key, offset=nil)
+    nodes = []
+    unless offset
+      offset = search_key.split("").map{|char| char[0,1].unpack('c')[0]}.inject(0) {|sum, i|  sum + i }
+    end
+    for i in (0...replicas)
+      zone = zones[(i + offset.to_i) % zones.length]
+      nodes << rings[zone].find_node(search_key)
+    end
+    nodes
+  end
+
+  def find_nodes_by_index(product_id, block_index)
+    offset = (product_id.to_i + block_index.to_i) % zones.length
+    search_key = "#{product_id}-#{block_index}"
+    find_nodes(search_key, offset)
+  end
+
+end

data/lib/clandestined/rendezvous_hash.rb

@@ -0,0 +1,37 @@
+require 'murmur3'
+
+class RendezvousHash
+
+  include Murmur3
+
+  attr_reader :nodes
+  attr_reader :murmur_seed
+  attr_reader :hash_function
+
+  def initialize(nodes=nil, murmur_seed=0, hash_function=method(:murmur3_32))
+    @nodes = nodes || []
+    @murmur_seed = murmur_seed
+
+    if hash_function == method(:murmur3_32)
+      @hash_function = lambda { |key| hash_function.call(key, murmur_seed) }
+    elsif murmur_seed != 0
+      raise ArgumentError, "Cannot apply seed to custom hash function #{hash_function}"
+    else
+      @hash_function = hash_function
+    end
+
+  end
+
+  def add_node(node)
+    @nodes.push(node) unless @nodes.include?(node)
+  end
+
+  def remove_node(node)
+    @nodes.delete(node) if @nodes.include?(node)
+  end
+
+  def find_node(key)
+    nodes.max {|a,b| hash_function.call("#{a}-#{key}") <=> hash_function.call("#{b}-#{key}")}
+  end
+
+end

data/lib/murmur3.rb

@@ -0,0 +1,9 @@
+begin
+  # Extension target, might not exist on some installations
+  require 'murmur3_native'
+  Murmur3 = Murmur3Native
+rescue LoadError
+  # Pure Ruby fallback, should cover all methods that are otherwise in extension
+  require 'murmur3_ruby'
+  Murmur3 = Murmur3Ruby
+end

data/lib/murmur3_ruby.rb

@@ -0,0 +1,50 @@
+module Murmur3Ruby
+  ## MurmurHash3 was written by Austin Appleby, and is placed in the public
+  ## domain. The author hereby disclaims copyright to this source code.
+
+  MASK32 = 0xffffffff
+
+  def murmur3_32_rotl(x, r)
+    ((x << r) | (x >> (32 - r))) & MASK32
+  end
+
+
+  def murmur3_32_fmix(h)
+    h &= MASK32
+    h ^= h >> 16
+    h = (h * 0x85ebca6b) & MASK32
+    h ^= h >> 13
+    h = (h * 0xc2b2ae35) & MASK32
+    h ^ (h >> 16)
+  end
+
+  def murmur3_32__mmix(k1)
+    k1 = (k1 * 0xcc9e2d51) & MASK32
+    k1 = murmur3_32_rotl(k1, 15)
+    (k1 * 0x1b873593) & MASK32
+  end
+
+  def murmur3_32(str, seed=0)
+    h1 = seed
+    numbers = str.unpack('V*C*')
+    tailn = str.length % 4
+    tail = numbers.slice!(numbers.size - tailn, tailn)
+    for k1 in numbers
+      h1 ^= murmur3_32__mmix(k1)
+      h1 = murmur3_32_rotl(h1, 13)
+      h1 = (h1*5 + 0xe6546b64) & MASK32
+    end
+
+    unless tail.empty?
+      k1 = 0
+      tail.reverse_each do |c1|
+        k1 = (k1 << 8) | c1
+      end
+      h1 ^= murmur3_32__mmix(k1)
+    end
+
+    h1 ^= str.length
+    murmur3_32_fmix(h1)
+  end
+
+end

data/test/test_cluster.rb

@@ -0,0 +1,462 @@
+
+require 'test/unit'
+require 'set'
+
+require 'clandestined/cluster'
+
+def my_hash_function(key)
+  Digest::MD5.hexdigest(key).to_i(16)
+end
+
+class ClusterTestCase < Test::Unit::TestCase
+
+  def test_init_no_options
+    cluster = Cluster.new()
+    assert_equal(1361238019, cluster.hash_function.call('6666'))
+    assert_equal(2, cluster.replicas)
+    assert_equal(Hash[], cluster.nodes)
+    assert_equal([], cluster.zones)
+    assert_equal(Hash[], cluster.zone_members)
+    assert_equal(Hash[], cluster.rings)
+  end
+
+  def test_murmur_seed
+    cluster = Cluster.new(nil, 2, 10)
+    assert_equal(2981722772, cluster.hash_function.call('6666'))
+  end
+
+  def test_custom_hash_function
+      cluster = Cluster.new(nil, 2, 0, method(:my_hash_function))
+      assert_equal(310130709337150341200260887719094037511, cluster.hash_function.call('6666'))
+  end
+
+  def test_seeded_custom_hash_function
+    assert_raises(ArgumentError) { Cluster.new(nil, 2, 10, method(:my_hash_function)) }
+  end
+
+  def test_init_single_zone
+    cluster_config = {
+      '1' => Hash[],
+      '2' => Hash[],
+      '3' => Hash[],
+    }
+    cluster = Cluster.new(cluster_config, 1)
+    assert_equal(1, cluster.replicas)
+    assert_equal(3, cluster.nodes.length)
+    assert_equal(1, cluster.zones.length)
+    assert_equal(3, cluster.zone_members[nil].length)
+    assert_equal(1, cluster.rings.length)
+    assert_equal(3, cluster.rings[nil].nodes.length)
+  end
+
+  def test_init_zones
+    cluster_config = {
+      '1' => {'zone' => 'a'},
+      '2' => {'zone' => 'b'},
+      '3' => {'zone' => 'a'},
+      '4' => {'zone' => 'b'},
+      '5' => {'zone' => 'a'},
+      '6' => {'zone' => 'b'},
+    }
+    cluster = Cluster.new(cluster_config)
+    assert_equal(2, cluster.replicas)
+    assert_equal(6, cluster.nodes.length)
+    assert_equal(['a', 'b'], cluster.zones)
+    assert_equal(['1', '3', '5'], cluster.zone_members['a'].sort)
+    assert_equal(['2', '4', '6'], cluster.zone_members['b'].sort)
+    assert_equal(2, cluster.rings.length)
+    assert_equal(3, cluster.rings['a'].nodes.length)
+    assert_equal(3, cluster.rings['b'].nodes.length)
+  end
+
+  def test_add_zone
+    cluster = Cluster.new()
+    assert_equal(0, cluster.nodes.length)
+    assert_equal([], cluster.zones)
+    assert_equal(0, cluster.zone_members.length)
+    assert_equal(0, cluster.rings.length)
+
+    cluster.add_zone('b')
+    assert_equal(0, cluster.nodes.length)
+    assert_equal(['b'], cluster.zones)
+    assert_equal(0, cluster.zone_members['b'].length)
+    assert_equal(0, cluster.rings.length)
+
+    cluster.add_zone('b')
+    assert_equal(0, cluster.nodes.length)
+    assert_equal(['b'], cluster.zones)
+    assert_equal(0, cluster.zone_members['b'].length)
+    assert_equal(0, cluster.rings.length)
+
+    cluster.add_zone('a')
+    assert_equal(0, cluster.nodes.length)
+    assert_equal(['a', 'b'], cluster.zones)
+    assert_equal(0, cluster.zone_members['a'].length)
+    assert_equal(0, cluster.zone_members['b'].length)
+    assert_equal(0, cluster.rings.length)
+  end
+
+  def test_add_node
+    cluster = Cluster.new()
+    assert_equal(0, cluster.nodes.length)
+    assert_equal([], cluster.zones)
+    assert_equal(0, cluster.zone_members.length)
+    assert_equal(0, cluster.rings.length)
+
+    cluster.add_node('2', 'b')
+    assert_equal(1, cluster.nodes.length)
+    assert_equal(['b'], cluster.zones)
+    assert_equal(1, cluster.zone_members.length)
+    assert_equal(['2'], cluster.zone_members['b'].sort)
+    assert_equal(1, cluster.rings.length)
+
+    cluster.add_node('1', 'a')
+    assert_equal(2, cluster.nodes.length)
+    assert_equal(['a', 'b'], cluster.zones)
+    assert_equal(2, cluster.zone_members.length)
+    assert_equal(['1'], cluster.zone_members['a'].sort)
+    assert_equal(['2'], cluster.zone_members['b'].sort)
+    assert_equal(2, cluster.rings.length)
+
+    cluster.add_node('21', 'b')
+    assert_equal(3, cluster.nodes.length)
+    assert_equal(['a', 'b'], cluster.zones)
+    assert_equal(2, cluster.zone_members.length)
+    assert_equal(['1'], cluster.zone_members['a'].sort)
+    assert_equal(['2', '21'], cluster.zone_members['b'].sort)
+    assert_equal(2, cluster.rings.length)
+
+    assert_raises(ArgumentError) {cluster.add_node('21')}
+    assert_raises(ArgumentError) {cluster.add_node('21', nil, nil)}
+    assert_raises(ArgumentError) {cluster.add_node('21', 'b', nil)}
+
+    cluster.add_node('22', 'c')
+    assert_equal(4, cluster.nodes.length)
+    assert_equal(['a', 'b', 'c'], cluster.zones)
+    assert_equal(3, cluster.zone_members.length)
+    assert_equal(['1'], cluster.zone_members['a'].sort)
+    assert_equal(['2', '21'], cluster.zone_members['b'].sort)
+    assert_equal(['22'], cluster.zone_members['c'].sort)
+    assert_equal(3, cluster.rings.length)
+  end
+
+  def test_remove_node
+    cluster_config = {
+      '1' => {'zone' => 'a'},
+      '2' => {'zone' => 'b'},
+      '3' => {'zone' => 'a'},
+      '4' => {'zone' => 'b'},
+      '5' => {'zone' => 'c'},
+      '6' => {'zone' => 'c'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    cluster.remove_node('4', 'b')
+    assert_equal(5, cluster.nodes.length)
+    assert_equal(['a', 'b', 'c'], cluster.zones)
+    assert_equal(3, cluster.zone_members.length)
+    assert_equal(['1', '3'], cluster.zone_members['a'].sort)
+    assert_equal(['2'], cluster.zone_members['b'].sort)
+    assert_equal(['5', '6'], cluster.zone_members['c'].sort)
+    assert_equal(3, cluster.rings.length)
+
+    cluster.remove_node('2', 'b')
+    assert_equal(4, cluster.nodes.length)
+    assert_equal(['a', 'c'], cluster.zones)
+    assert_equal(2, cluster.zone_members.length)
+    assert_equal(['1', '3'], cluster.zone_members['a'].sort)
+    assert_equal(nil, cluster.zone_members['b'])
+    assert_equal(['5', '6'], cluster.zone_members['c'].sort)
+    assert_equal(2, cluster.rings.length)
+  end
+
+  def test_node_name_by_id
+    cluster_config = {
+      '1' => {'name' => 'node1', 'zone' => 'a'},
+      '2' => {'name' => 'node2', 'zone' => 'b'},
+      '3' => {'name' => 'node3', 'zone' => 'a'},
+      '4' => {'name' => 'node4', 'zone' => 'b'},
+      '5' => {'name' => 'node5', 'zone' => 'c'},
+      '6' => {'name' => 'node6', 'zone' => 'c'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    assert_equal('node1', cluster.node_name('1'))
+    assert_equal('node2', cluster.node_name('2'))
+    assert_equal('node3', cluster.node_name('3'))
+    assert_equal('node4', cluster.node_name('4'))
+    assert_equal('node5', cluster.node_name('5'))
+    assert_equal('node6', cluster.node_name('6'))
+    assert_equal(nil, cluster.node_name('7'))
+  end
+
+  def test_find_nodes
+    cluster_config = {
+      '1' => {'name' => 'node1', 'zone' => 'a'},
+      '2' => {'name' => 'node2', 'zone' => 'a'},
+      '3' => {'name' => 'node3', 'zone' => 'b'},
+      '4' => {'name' => 'node4', 'zone' => 'b'},
+      '5' => {'name' => 'node5', 'zone' => 'c'},
+      '6' => {'name' => 'node6', 'zone' => 'c'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    assert_equal(['2', '3'], cluster.find_nodes('lol'))
+    assert_equal(['6', '2'], cluster.find_nodes('wat'))
+    assert_equal(['5', '2'], cluster.find_nodes('ok'))
+    assert_equal(['1', '3'], cluster.find_nodes('bar'))
+    assert_equal(['1', '3'], cluster.find_nodes('foo'))
+    assert_equal(['2', '4'], cluster.find_nodes('slap'))
+  end
+
+  def test_find_nodes_by_index
+    cluster_config = {
+      '1' => {'name' => 'node1', 'zone' => 'a'},
+      '2' => {'name' => 'node2', 'zone' => 'a'},
+      '3' => {'name' => 'node3', 'zone' => 'b'},
+      '4' => {'name' => 'node4', 'zone' => 'b'},
+      '5' => {'name' => 'node5', 'zone' => 'c'},
+      '6' => {'name' => 'node6', 'zone' => 'c'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    assert_equal(['6', '1'], cluster.find_nodes_by_index(1, 1))
+    assert_equal(['2', '4'], cluster.find_nodes_by_index(1, 2))
+    assert_equal(['4', '5'], cluster.find_nodes_by_index(1, 3))
+    assert_equal(['1', '4'], cluster.find_nodes_by_index(2, 1))
+    assert_equal(['3', '5'], cluster.find_nodes_by_index(2, 2))
+    assert_equal(['5', '2'], cluster.find_nodes_by_index(2, 3))
+  end
+
+end
+
+
+class ClusterIntegrationTestCase < Test::Unit::TestCase
+
+  def test_grow
+    cluster_config = {
+      '1' => {'name' => 'node1', 'zone' => 'a'},
+      '2' => {'name' => 'node2', 'zone' => 'a'},
+      '3' => {'name' => 'node3', 'zone' => 'b'},
+      '4' => {'name' => 'node4', 'zone' => 'b'},
+      '5' => {'name' => 'node5', 'zone' => 'c'},
+      '6' => {'name' => 'node6', 'zone' => 'c'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    placements = Hash[]
+    for i in cluster.nodes.keys
+      placements[i] = []
+    end
+    for i in (0..1000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        placements[node].push(i)
+      end
+    end
+
+    cluster.add_node('7', 'a', 'node7')
+    cluster.add_node('8', 'b', 'node8')
+    cluster.add_node('9', 'c', 'node9')
+
+    new_placements = Hash[]
+    for i in cluster.nodes.keys
+      new_placements[i] = []
+    end
+    for i in (0..1000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        new_placements[node].push(i)
+      end
+    end
+
+    keys = placements.values.flatten
+    new_keys = new_placements.values.flatten
+    assert_equal(keys.sort, new_keys.sort)
+
+    added = 0
+    removed = 0
+    new_placements.each do |node, assignments|
+      after = assignments.to_set
+      before = placements.fetch(node, []).to_set
+      removed += before.difference(after).length
+      added += after.difference(before).length
+    end
+
+    assert_equal(added, removed)
+    assert_equal(1384, (added + removed))
+  end
+
+  def test_shrink
+    cluster_config = {
+      '1' => {'name' => 'node1', 'zone' => 'a'},
+      '2' => {'name' => 'node2', 'zone' => 'a'},
+      '3' => {'name' => 'node3', 'zone' => 'b'},
+      '4' => {'name' => 'node4', 'zone' => 'b'},
+      '5' => {'name' => 'node5', 'zone' => 'c'},
+      '6' => {'name' => 'node6', 'zone' => 'c'},
+      '7' => {'name' => 'node7', 'zone' => 'a'},
+      '8' => {'name' => 'node8', 'zone' => 'a'},
+      '9' => {'name' => 'node9', 'zone' => 'b'},
+      '10' => {'name' => 'node10', 'zone' => 'b'},
+      '11' => {'name' => 'node11', 'zone' => 'c'},
+      '12' => {'name' => 'node12', 'zone' => 'c'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    placements = Hash[]
+    for i in cluster.nodes.keys
+      placements[i] = []
+    end
+    for i in (0...10000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        placements[node].push(i)
+      end
+    end
+
+    cluster.remove_node('7', 'a', 'nodee7')
+    cluster.remove_node('9', 'b', 'node9')
+    cluster.remove_node('11', 'c', 'node11')
+
+    new_placements = Hash[]
+    for i in cluster.nodes.keys
+      new_placements[i] = []
+    end
+    for i in (0...10000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        new_placements[node].push(i)
+      end
+    end
+
+    keys = placements.values.flatten
+    new_keys = new_placements.values.flatten
+    assert_equal(keys.sort, new_keys.sort)
+
+    added = 0
+    removed = 0
+    placements.each do |node, assignments|
+      after = assignments.to_set
+      before = new_placements.fetch(node, []).to_set
+      removed += before.difference(after).length
+      added += after.difference(before).length
+    end
+
+    assert_equal(added, removed)
+    assert_equal(9804, (added + removed))
+  end
+
+  def test_add_zone
+    cluster_config = {
+      '1' => {'name' => 'node1', 'zone' => 'a'},
+      '2' => {'name' => 'node2', 'zone' => 'a'},
+      '3' => {'name' => 'node3', 'zone' => 'b'},
+      '4' => {'name' => 'node4', 'zone' => 'b'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    placements = Hash[]
+    for i in cluster.nodes.keys
+      placements[i] = []
+    end
+    for i in (0...1000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        placements[node].push(i)
+      end
+    end
+
+    cluster.add_node('5', 'c', 'node5')
+    cluster.add_node('6', 'c', 'node6')
+
+    new_placements = Hash[]
+    for i in cluster.nodes.keys
+      new_placements[i] = []
+    end
+    for i in (0...1000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        new_placements[node].push(i)
+      end
+    end
+
+    keys = placements.values.flatten
+    new_keys = new_placements.values.flatten
+    assert_equal(keys.sort, new_keys.sort)
+
+    added = 0
+    removed = 0
+    new_placements.each do |node, assignments|
+      after = assignments.to_set
+      before = placements.fetch(node, []).to_set
+      removed += before.difference(after).length
+      added += after.difference(before).length
+    end
+
+    assert_equal(added, removed)
+    assert_equal(1332, (added + removed))
+  end
+
+  def test_remove_zone
+    cluster_config = {
+      '1' => {'name' => 'node1', 'zone' => 'a'},
+      '2' => {'name' => 'node2', 'zone' => 'a'},
+      '3' => {'name' => 'node3', 'zone' => 'b'},
+      '4' => {'name' => 'node4', 'zone' => 'b'},
+      '5' => {'name' => 'node5', 'zone' => 'c'},
+      '6' => {'name' => 'node6', 'zone' => 'c'},
+      '7' => {'name' => 'node7', 'zone' => 'a'},
+      '8' => {'name' => 'node8', 'zone' => 'a'},
+      '9' => {'name' => 'node9', 'zone' => 'b'},
+      '10' => {'name' => 'node10', 'zone' => 'b'},
+      '11' => {'name' => 'node11', 'zone' => 'c'},
+      '12' => {'name' => 'node12', 'zone' => 'c'},
+    }
+    cluster = Cluster.new(cluster_config)
+
+    placements = Hash[]
+    for i in cluster.nodes.keys
+      placements[i] = []
+    end
+    for i in (0...10000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        placements[node].push(i)
+      end
+    end
+
+    cluster.remove_node('5', 'c', 'node5')
+    cluster.remove_node('6', 'c', 'node6')
+    cluster.remove_node('11', 'c', 'node11')
+    cluster.remove_node('12', 'c', 'node12')
+
+    new_placements = Hash[]
+    for i in cluster.nodes.keys
+      new_placements[i] = []
+    end
+    for i in (0...10000)
+      nodes = cluster.find_nodes(i.to_s)
+      for node in nodes
+        new_placements[node].push(i)
+      end
+    end
+
+    keys = placements.values.flatten
+    new_keys = new_placements.values.flatten
+    assert_equal(keys.sort, new_keys.sort)
+
+    added = 0
+    removed = 0
+    placements.each do |node, assignments|
+      after = assignments.to_set
+      before = new_placements.fetch(node, []).to_set
+      removed += before.difference(after).length
+      added += after.difference(before).length
+    end
+
+    assert_equal(added, removed)
+    assert_equal(13332, (added + removed))
+  end
+
+end

data/test/test_rendezvous_hash.rb

@@ -0,0 +1,180 @@
+
+require 'test/unit'
+require 'set'
+
+require 'clandestined/rendezvous_hash'
+
+def my_hash_function(key)
+  Digest::MD5.hexdigest(key).to_i(16)
+end
+
+class RendezvousHashTestCase < Test::Unit::TestCase
+
+  def test_init_no_options
+    rendezvous = RendezvousHash.new()
+    assert_equal(0, rendezvous.nodes.length)
+    assert_equal(1361238019, rendezvous.hash_function.call('6666'))
+  end
+
+  def test_init
+    nodes = ['0', '1', '2']
+    rendezvous = RendezvousHash.new(nodes)
+    assert_equal(3, rendezvous.nodes.length)
+    assert_equal(1361238019, rendezvous.hash_function.call('6666'))
+  end
+
+  def test_murmur_seed
+    rendezvous = RendezvousHash.new(nil, 10)
+    assert_equal(2981722772, rendezvous.hash_function.call('6666'))
+  end
+
+  def test_custom_hash_function
+      rendezvous = RendezvousHash.new(nil, 0, method(:my_hash_function))
+      assert_equal(310130709337150341200260887719094037511, rendezvous.hash_function.call('6666'))
+  end
+
+  def test_seeded_custom_hash_function
+    assert_raises(ArgumentError) { RendezvousHash.new(nil, 10, method(:my_hash_function)) }
+  end
+
+  def test_add_node
+    rendezvous = RendezvousHash.new()
+    rendezvous.add_node('1')
+    assert_equal(1, rendezvous.nodes.length)
+    rendezvous.add_node('1')
+    assert_equal(1, rendezvous.nodes.length)
+    rendezvous.add_node('2')
+    assert_equal(2, rendezvous.nodes.length)
+    rendezvous.add_node('1')
+    assert_equal(2, rendezvous.nodes.length)
+  end
+
+  def test_remove_node
+    nodes = ['0', '1', '2']
+    rendezvous = RendezvousHash.new(nodes)
+    rendezvous.remove_node('2')
+    assert_equal(2, rendezvous.nodes.length)
+    rendezvous.remove_node('2')
+    assert_equal(2, rendezvous.nodes.length)
+    rendezvous.remove_node('1')
+    assert_equal(1, rendezvous.nodes.length)
+    rendezvous.remove_node('0')
+    assert_equal(0, rendezvous.nodes.length)
+  end
+
+  def test_find_node
+    nodes = ['0', '1', '2']
+    rendezvous = RendezvousHash.new(nodes)
+    assert_equal('0', rendezvous.find_node('ok'))
+    assert_equal('1', rendezvous.find_node('mykey'))
+    assert_equal('2', rendezvous.find_node('wat'))
+  end
+
+  def test_find_node_after_removal
+    nodes = ['0', '1', '2']
+    rendezvous = RendezvousHash.new(nodes)
+    rendezvous.remove_node('1')
+    assert_equal('0', rendezvous.find_node('ok'))
+    assert_equal('0', rendezvous.find_node('mykey'))
+    assert_equal('2', rendezvous.find_node('wat'))
+  end
+
+  def test_find_node_after_addition
+    nodes = ['0', '1', '2']
+    rendezvous = RendezvousHash.new(nodes)
+    assert_equal('0', rendezvous.find_node('ok'))
+    assert_equal('1', rendezvous.find_node('mykey'))
+    assert_equal('2', rendezvous.find_node('wat'))
+    assert_equal('2', rendezvous.find_node('lol'))
+    rendezvous.add_node('3')
+    assert_equal('0', rendezvous.find_node('ok'))
+    assert_equal('1', rendezvous.find_node('mykey'))
+    assert_equal('2', rendezvous.find_node('wat'))
+    assert_equal('3', rendezvous.find_node('lol'))
+  end
+
+end
+
+class RendezvousHashIntegrationTestCase < Test::Unit::TestCase
+
+  def test_grow
+    rendezvous = RendezvousHash.new()
+
+    placements = Hash[]
+    for i in (0...10)
+      rendezvous.add_node(i.to_s)
+      placements[i.to_s] = []
+    end
+    for i in (0...1000)
+      node = rendezvous.find_node(i.to_s)
+      placements[node].push(i)
+    end
+
+    new_placements = Hash[]
+    for i in (0...20)
+      rendezvous.add_node(i.to_s)
+      new_placements[i.to_s] = []
+    end
+    for i in (0...1000)
+      node = rendezvous.find_node(i.to_s)
+      new_placements[node].push(i)
+    end
+
+    keys = placements.values.flatten
+    new_keys = new_placements.values.flatten
+    assert_equal(keys.sort, new_keys.sort)
+
+    added = 0
+    removed = 0
+    new_placements.each do |node, assignments|
+      after = assignments.to_set
+      before = placements.fetch(node, []).to_set
+      removed += before.difference(after).length
+      added += after.difference(before).length
+    end
+
+    assert_equal(added, removed)
+    assert_equal(1062, (added + removed))
+  end
+
+  def test_shrink
+    rendezvous = RendezvousHash.new()
+
+    placements = {}
+    for i in (0...10)
+      rendezvous.add_node(i.to_s)
+      placements[i.to_s] = []
+    end
+    for i in (0...1000)
+      node = rendezvous.find_node(i.to_s)
+      placements[node].push(i)
+    end
+
+    rendezvous.remove_node('9')
+    new_placements = {}
+    for i in (0...9)
+      new_placements[i.to_s] = []
+    end
+    for i in (0...1000)
+      node = rendezvous.find_node(i.to_s)
+      new_placements[node].push(i)
+    end
+
+    keys = placements.values.flatten
+    new_keys = new_placements.values.flatten
+    assert_equal(keys.sort, new_keys.sort)
+
+    added = 0
+    removed = 0
+    placements.each do |node, assignments|
+      after = assignments.to_set
+      before = new_placements.fetch(node, []).to_set
+      removed += before.difference(after).length
+      added += after.difference(before).length
+    end
+
+    assert_equal(added, removed)
+    assert_equal(202, (added + removed))
+  end
+
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: clandestined
+version: !ruby/object:Gem::Version
+  version: 1.0.0a
+platform: ruby
+authors:
+- Ernest W. Durbin III
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.6
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+- !ruby/object:Gem::Dependency
+  name: rubydoctest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.3
+description: |
+  rendezvous hashing implementation based on murmur3 hash
+email: ewdurbin@gmail.com
+executables: []
+extensions:
+- ext/murmur3_native/extconf.rb
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- clandestined.gemspec
+- ext/murmur3_native/extconf.rb
+- ext/murmur3_native/murmur3_native.c
+- lib/clandestined/cluster.rb
+- lib/clandestined/rendezvous_hash.rb
+- lib/murmur3.rb
+- lib/murmur3_ruby.rb
+- test/test_cluster.rb
+- test/test_rendezvous_hash.rb
+homepage: https://github.com/ewdurbin/clandestined-ruby
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: rendezvous hashing implementation based on murmur3 hash
+test_files:
+- test/test_cluster.rb
+- test/test_rendezvous_hash.rb