fast_hash_ring 0.1

data/CREDITS

@@ -0,0 +1,12 @@
+======================================
+= fast_hash_ring credits                  =
+======================================
+
+Original Author (Python Implementation):
+  Amir Salihefendic http://amix.dk
+
+Ported to Ruby by:
+  Mitchell Hashimoto http://mitchellhashimoto.com
+
+Ported to C Ruby Extension by:
+  Flinn Mueller http://actsasflinn.com

data/LICENSE

@@ -0,0 +1,10 @@
+Copyright (c) 2009, Flinn Mueller, Mitchell Hashimoto
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    * Neither the name of the owner nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.rdoc

@@ -0,0 +1,31 @@
+= fast_hash_ring
+
+== Background
+
+fast_hash_ring is a Ruby C extension implementation of consistent hashing. 
+fast_hash_ring is based on the Ruby code Mitchell Hashimoto ported from 
+original Python code written by Amir Salihefendic. A comprehensive blog 
+post detailing the methods and reasoning for such a library can be viewed 
+by visiting the following URL:
+
+http://amix.dk/blog/viewEntry/19367
+
+== Usage
+
+  require 'rubygems'
+  require 'fast_hash_ring'
+
+  memcache_servers = ['192.168.0.111:14107',
+                      '192.168.0.112:14107',
+                      '192.168.0.113:14108']
+ 
+  # Since server 1 has double the RAM, lets weight it
+  # twice as much to get twice the keys. This is optional
+  weights = { '192.168.0.111' => 2 }
+
+  ring = FastHashRing.new(memcache_servers, weights)
+  server = ring.get_node('my_key')
+
+== Installation
+
+  sudo gem install fast_hash_ring --source=http://gemcutter.org

data/Rakefile

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+$root = Pathname(__FILE__).dirname
+
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+task :spec do
+  load $root.join('spec', 'spec_base.rb')
+end
+task :default => [ :spec ]
+
+CLEAN.include('pkg', 'tmp')
+
+gemspec = Gem::Specification.new do |s|
+  s.name = 'fast_hash_ring'
+  s.version = '0.1'
+  s.authors = [ 'Flinn' ]
+  s.email = 'flinn@actsasflinn.com'
+  s.homepage = 'http://github.com/actsasflinn/fast_hash_ring/'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'A Ruby C Extension based on the pure Ruby HashRing gem'
+  s.require_path = 'ext'
+  s.test_file = 'spec/spec_base.rb'
+  s.has_rdoc = true
+  s.extra_rdoc_files = %w{ README.rdoc }
+
+  s.files = ['CREDITS',
+             'LICENSE',
+             'Rakefile',
+             'README.rdoc'] +
+             Dir['ext/**/*.[rb|c|h]'] +
+             Dir['spec/**/*'] +
+             Dir['benchmarks/**/*']
+  s.extensions << "ext/extconf.rb"
+end
+
+task :gemspec do
+  File.open('fast_hash_ring.gemspec', 'w') do |f|
+    f.write(gemspec.to_ruby)
+  end
+end
+
+Rake::GemPackageTask.new(gemspec) do |pkg|
+  pkg.need_tar = true
+end
+
+Rake::PackageTask.new('fast_hash_ring', '0.1') do |pkg|
+  pkg.need_zip = true
+  pkg.package_files = FileList[
+    'COPYING',
+    'Rakefile',
+    'README.rdoc',
+    'ext/**/*',
+    'spec/**/*',
+    'benchmarks/**/*'
+  ].to_a
+  class << pkg
+    def package_name
+      "#{@name}-#{@version}-src"
+    end
+  end
+end

data/benchmarks/fast_hash_ring.rb

@@ -0,0 +1,32 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+require 'hash_ring'
+require 'fast_hash_ring'
+
+servers = []
+16.times do |i|
+  servers << "127.0.0.#{i+1}:12345"
+end
+
+weights = {}
+4.times do |i|
+  weights["127.0.0.#{i+1}:12345"] = 8
+end
+
+keys = []
+10_000.times do |i|
+  keys << Faker::Name.name
+end
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  h = HashRing.new(servers, weights)
+  b.report('Hash Ring') do
+    keys.each{ |key| server = h.get_node(key) }
+  end
+
+  fh = FastHashRing.new(servers, weights)
+  b.report('Fash Hash Ring') do
+    keys.each{ |key| server = fh.get_node(key) }
+  end
+end

data/ext/extconf.rb

@@ -0,0 +1,30 @@
+# ENV["RC_ARCHS"] = `uname -m`.chomp if `uname -sr` =~ /^Darwin/
+#
+# require 'mkmf'
+#
+
+if RUBY_PLATFORM =~ /darwin/
+  ENV["RC_ARCHS"] = `uname -m`.chomp if `uname -sr` =~ /^Darwin/
+
+  ENV["RC_ARCHS"] = "x86_64" if RUBY_PLATFORM =~ /darwin10.0/
+
+  # On PowerPC the defaults are fine
+  ENV["RC_ARCHS"] = '' if `uname -m` =~ /^Power Macintosh/
+end
+
+# Loads mkmf which is used to make makefiles for Ruby extensions
+require 'mkmf'
+
+# Give it a name
+extension_name = 'fast_hash_ring'
+
+dir_config("fast_hash_ring")
+
+# NOTE: use GCC flags unless Visual C compiler is used
+$CFLAGS << ' -Wall ' unless RUBY_PLATFORM =~ /mswin/
+
+if RUBY_VERSION < '1.8.6'
+  $CFLAGS << ' -DRUBY_LESS_THAN_186'
+end
+
+create_makefile 'fast_hash_ring'

data/ext/fast_hash_ring.c

@@ -0,0 +1,192 @@
+#include <fast_hash_ring.h>
+
+void hash_digest(char *key, unsigned char md5pword[16]){
+  md5_state_t md5state;
+
+  md5_init(&md5state);
+  md5_append(&md5state, (unsigned char *)key, strlen(key));
+  md5_finish(&md5state, md5pword);
+}
+
+unsigned int hash_val(char *key, int x){
+  unsigned char digest[16];
+
+  hash_digest(key, digest);
+  return ((digest[3 + x] << 24) |
+          (digest[2 + x] << 16) |
+          (digest[1 + x] <<  8) |
+           digest[0 + x]);
+}
+
+static VALUE cFastHashRing_generate_circle(VALUE vself){
+  VALUE vnodes, vweights, vring, vsorted_keys;
+  int i, j, n, vnodes_len;
+  int total_weight = 0;
+
+  vnodes = rb_iv_get(vself, "@nodes");
+  vweights = rb_iv_get(vself, "@weights");
+  vring = rb_iv_get(vself, "@ring");
+  vsorted_keys = rb_iv_get(vself, "@sorted_keys");
+
+  vnodes_len = RARRAY_LEN(vnodes);
+  for(i = 0;i < vnodes_len;i++){
+    VALUE vnode = rb_ary_entry(vnodes, i);
+    VALUE vweight = rb_hash_aref(vweights, vnode);
+
+    if(vweight == Qnil) vweight = INT2NUM(1);
+    total_weight += NUM2INT(vweight);
+  }
+
+  for(i = 0;i < vnodes_len;i++){
+    VALUE vnode = rb_ary_entry(vnodes, i);
+    VALUE vweight = rb_hash_aref(vweights, vnode);
+
+    if(vweight == Qnil) vweight = INT2NUM(1);
+    int weight = NUM2INT(vweight);
+
+    int factor = floor((40 * vnodes_len * weight) / total_weight);
+    for(j = 0;j < factor;j++){
+      int blen = 0;
+      VALUE vstring = rb_convert_type(vnode, T_STRING, "String", "to_s");
+      blen += strlen(RSTRING_PTR(vstring));
+      blen += 12;
+      char key[blen];
+      sprintf(key, "%s-%d", RSTRING_PTR(vstring), j);
+
+      for(n=0;n<3;n++){
+        unsigned int hkey = hash_val(key, n*4);
+        rb_hash_aset(vring, INT2NUM(hkey), vnode);
+        rb_ary_push(vsorted_keys, INT2NUM(hkey));
+      }
+    }
+  }
+  rb_iv_set(vself, "@sorted_keys", rb_ary_sort(vsorted_keys));
+
+  return Qtrue;
+}
+
+static VALUE cFastHashRing_gen_key(VALUE vself, VALUE vstring_key){
+  unsigned int key = hash_val(RSTRING_PTR(vstring_key), 0);
+
+  return INT2NUM(key);
+}
+
+int bisect(VALUE vsorted_keys, VALUE vkey){
+  int i, vsorted_keys_len;
+
+  vsorted_keys_len = RARRAY_LEN(vsorted_keys);
+  for(i=0;i<vsorted_keys_len;i++){
+    VALUE vsorted_key = rb_ary_entry(vsorted_keys, i);
+    if (NUM2LONG(vkey) < NUM2LONG(vsorted_key)) return i;
+  }
+
+  return vsorted_keys_len;
+}
+
+static VALUE cFastHashRing_get_node_pos(VALUE vself, VALUE vstring_key){
+  VALUE vring, vsorted_keys, vkey;
+  int pos;
+
+  vring = rb_iv_get(vself, "@ring");
+
+  if(rb_funcall(vring, rb_intern("empty?"), 0)) return Qnil;
+
+  vsorted_keys = rb_iv_get(vself, "@sorted_keys");
+
+  vkey = cFastHashRing_gen_key(vself, vstring_key);
+  pos = bisect(vsorted_keys, vkey);
+
+  if(pos == RARRAY_LEN(vsorted_keys))
+    return INT2NUM(0);
+  else
+    return INT2NUM(pos);
+}
+
+static VALUE cFastHashRing_get_node(VALUE vself, VALUE vstring_key){
+  VALUE vpos, vring, vsorted_keys, vkey, vnode;
+  vpos = cFastHashRing_get_node_pos(vself, vstring_key);
+
+  if(vpos == Qnil) return Qnil;
+
+  vring = rb_iv_get(vself, "@ring");
+  vsorted_keys = rb_iv_get(vself, "@sorted_keys");
+
+  vkey = rb_ary_entry(vsorted_keys, NUM2INT(vpos));
+  vnode = rb_hash_aref(vring, vkey);
+
+  return vnode;
+}
+
+static VALUE cFastHashRing_iterate_nodes(VALUE vself, VALUE vstring_key){
+  VALUE vreturned_values, vpos, vring, vsorted_keys;
+  int i;
+
+  vreturned_values = rb_ary_new();
+  vpos = cFastHashRing_get_node_pos(vself, vstring_key);
+
+  vring = rb_iv_get(vself, "@ring");
+  vsorted_keys = rb_iv_get(vself, "@sorted_keys");
+
+  for(i=0;i<RARRAY_LEN(vsorted_keys);i++){
+    VALUE vkey = rb_ary_entry(vsorted_keys, i);
+    VALUE vvalue = rb_hash_aref(vring, vkey);
+    if(!(rb_ary_includes(vreturned_values, vvalue)))
+      rb_ary_push(vreturned_values, vvalue);
+  }
+
+  for(i=0;i<RARRAY_LEN(vsorted_keys);i++){
+    if(i >= NUM2INT(vpos)) break;
+
+    VALUE vkey = rb_ary_entry(vsorted_keys, i);
+    VALUE vvalue = rb_hash_aref(vring, vkey);
+
+    if(rb_ary_includes(vreturned_values, vvalue)) continue;
+
+    rb_ary_push(vreturned_values, vvalue);
+  }
+
+  return vreturned_values;
+}
+
+static VALUE cFastHashRing_initialize(int argc, VALUE *argv, VALUE vself){
+  VALUE vnodes, vweights, vring, vsorted_keys;
+
+  rb_scan_args(argc, argv, "11", &vnodes, &vweights);
+
+  if(TYPE(vnodes) != T_ARRAY) vnodes = rb_ary_new3(1, vnodes);
+  Check_Type(vnodes, T_ARRAY);
+  if(NIL_P(vweights)) vweights = rb_hash_new();
+  vring = rb_hash_new();
+  vsorted_keys = rb_ary_new();
+
+  rb_iv_set(vself, "@nodes", vnodes);
+  rb_iv_set(vself, "@weights", vweights);
+
+  rb_iv_set(vself, "@ring", vring);
+  rb_iv_set(vself, "@sorted_keys", vsorted_keys);
+
+  return cFastHashRing_generate_circle(vself);
+}
+
+static VALUE cFastHashRing_sorted_keys(VALUE vself){
+  return rb_iv_get(vself, "@sorted_keys");
+}
+
+static VALUE cFastHashRing_nodes(VALUE vself){
+  return rb_iv_get(vself, "@nodes");
+}
+
+VALUE cFastHashRing;
+
+void Init_fast_hash_ring(){
+  cFastHashRing = rb_define_class("FastHashRing", rb_cObject);
+
+  rb_define_private_method(cFastHashRing, "initialize", cFastHashRing_initialize, -1);
+  rb_define_private_method(cFastHashRing, "generate_circle", cFastHashRing_generate_circle, -1);
+  rb_define_method(cFastHashRing, "get_node", cFastHashRing_get_node, 1);
+  rb_define_method(cFastHashRing, "get_node_pos", cFastHashRing_get_node_pos, 1);
+  rb_define_method(cFastHashRing, "iterate_nodes", cFastHashRing_iterate_nodes, 1);
+  rb_define_method(cFastHashRing, "gen_key", cFastHashRing_gen_key, 1);
+  rb_define_method(cFastHashRing, "sorted_keys", cFastHashRing_sorted_keys, 0);
+  rb_define_method(cFastHashRing, "nodes", cFastHashRing_nodes, 0);
+}

data/ext/fast_hash_ring.h

@@ -0,0 +1,11 @@
+#ifndef RUBY_FASTHASHRING
+#define RUBY_FASTHASHRING
+
+#include <ruby.h>
+#include <math.h>
+#include <stdio.h>
+#include "md5.h"
+
+void Init_fast_hash_ring();
+
+#endif

data/ext/md5.c

@@ -0,0 +1,381 @@
+/*
+  Copyright (C) 1999, 2000, 2002 Aladdin Enterprises.  All rights reserved.
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  L. Peter Deutsch
+  ghost@aladdin.com
+
+ */
+/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */
+/*
+  Independent implementation of MD5 (RFC 1321).
+
+  This code implements the MD5 Algorithm defined in RFC 1321, whose
+  text is available at
+	http://www.ietf.org/rfc/rfc1321.txt
+  The code is derived from the text of the RFC, including the test suite
+  (section A.5) but excluding the rest of Appendix A.  It does not include
+  any code or documentation that is identified in the RFC as being
+  copyrighted.
+
+  The original and principal author of md5.c is L. Peter Deutsch
+  <ghost@aladdin.com>.  Other authors are noted in the change history
+  that follows (in reverse chronological order):
+
+  2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order
+	either statically or dynamically; added missing #include <string.h>
+	in library.
+  2002-03-11 lpd Corrected argument list for main(), and added int return
+	type, in test program and T value program.
+  2002-02-21 lpd Added missing #include <stdio.h> in test program.
+  2000-07-03 lpd Patched to eliminate warnings about "constant is
+	unsigned in ANSI C, signed in traditional"; made test program
+	self-checking.
+  1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
+  1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5).
+  1999-05-03 lpd Original version.
+ */
+
+#include "md5.h"
+#include <string.h>
+
+#undef BYTE_ORDER	/* 1 = big-endian, -1 = little-endian, 0 = unknown */
+#ifdef ARCH_IS_BIG_ENDIAN
+#  define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1)
+#else
+#  define BYTE_ORDER 0
+#endif
+
+#define T_MASK ((md5_word_t)~0)
+#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87)
+#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9)
+#define T3    0x242070db
+#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111)
+#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050)
+#define T6    0x4787c62a
+#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec)
+#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe)
+#define T9    0x698098d8
+#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850)
+#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e)
+#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841)
+#define T13    0x6b901122
+#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c)
+#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71)
+#define T16    0x49b40821
+#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d)
+#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf)
+#define T19    0x265e5a51
+#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855)
+#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2)
+#define T22    0x02441453
+#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e)
+#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437)
+#define T25    0x21e1cde6
+#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829)
+#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278)
+#define T28    0x455a14ed
+#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa)
+#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07)
+#define T31    0x676f02d9
+#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375)
+#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd)
+#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e)
+#define T35    0x6d9d6122
+#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3)
+#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb)
+#define T38    0x4bdecfa9
+#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f)
+#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f)
+#define T41    0x289b7ec6
+#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805)
+#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a)
+#define T44    0x04881d05
+#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6)
+#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a)
+#define T47    0x1fa27cf8
+#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a)
+#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb)
+#define T50    0x432aff97
+#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58)
+#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6)
+#define T53    0x655b59c3
+#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d)
+#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82)
+#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e)
+#define T57    0x6fa87e4f
+#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f)
+#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb)
+#define T60    0x4e0811a1
+#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d)
+#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca)
+#define T63    0x2ad7d2bb
+#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e)
+
+
+static void
+md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/)
+{
+    md5_word_t
+	a = pms->abcd[0], b = pms->abcd[1],
+	c = pms->abcd[2], d = pms->abcd[3];
+    md5_word_t t;
+#if BYTE_ORDER > 0
+    /* Define storage only for big-endian CPUs. */
+    md5_word_t X[16];
+#else
+    /* Define storage for little-endian or both types of CPUs. */
+    md5_word_t xbuf[16];
+    const md5_word_t *X;
+#endif
+
+    {
+#if BYTE_ORDER == 0
+	/*
+	 * Determine dynamically whether this is a big-endian or
+	 * little-endian machine, since we can use a more efficient
+	 * algorithm on the latter.
+	 */
+	static const int w = 1;
+
+	if (*((const md5_byte_t *)&w)) /* dynamic little-endian */
+#endif
+#if BYTE_ORDER <= 0		/* little-endian */
+	{
+	    /*
+	     * On little-endian machines, we can process properly aligned
+	     * data without copying it.
+	     */
+	    if (!((data - (const md5_byte_t *)0) & 3)) {
+		/* data are properly aligned */
+		X = (const md5_word_t *)data;
+	    } else {
+		/* not aligned */
+		memcpy(xbuf, data, 64);
+		X = xbuf;
+	    }
+	}
+#endif
+#if BYTE_ORDER == 0
+	else			/* dynamic big-endian */
+#endif
+#if BYTE_ORDER >= 0		/* big-endian */
+	{
+	    /*
+	     * On big-endian machines, we must arrange the bytes in the
+	     * right order.
+	     */
+	    const md5_byte_t *xp = data;
+	    int i;
+
+#  if BYTE_ORDER == 0
+	    X = xbuf;		/* (dynamic only) */
+#  else
+#    define xbuf X		/* (static only) */
+#  endif
+	    for (i = 0; i < 16; ++i, xp += 4)
+		xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24);
+	}
+#endif
+    }
+
+#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
+
+    /* Round 1. */
+    /* Let [abcd k s i] denote the operation
+       a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */
+#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
+#define SET(a, b, c, d, k, s, Ti)\
+  t = a + F(b,c,d) + X[k] + Ti;\
+  a = ROTATE_LEFT(t, s) + b
+    /* Do the following 16 operations. */
+    SET(a, b, c, d,  0,  7,  T1);
+    SET(d, a, b, c,  1, 12,  T2);
+    SET(c, d, a, b,  2, 17,  T3);
+    SET(b, c, d, a,  3, 22,  T4);
+    SET(a, b, c, d,  4,  7,  T5);
+    SET(d, a, b, c,  5, 12,  T6);
+    SET(c, d, a, b,  6, 17,  T7);
+    SET(b, c, d, a,  7, 22,  T8);
+    SET(a, b, c, d,  8,  7,  T9);
+    SET(d, a, b, c,  9, 12, T10);
+    SET(c, d, a, b, 10, 17, T11);
+    SET(b, c, d, a, 11, 22, T12);
+    SET(a, b, c, d, 12,  7, T13);
+    SET(d, a, b, c, 13, 12, T14);
+    SET(c, d, a, b, 14, 17, T15);
+    SET(b, c, d, a, 15, 22, T16);
+#undef SET
+
+     /* Round 2. */
+     /* Let [abcd k s i] denote the operation
+          a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */
+#define G(x, y, z) (((x) & (z)) | ((y) & ~(z)))
+#define SET(a, b, c, d, k, s, Ti)\
+  t = a + G(b,c,d) + X[k] + Ti;\
+  a = ROTATE_LEFT(t, s) + b
+     /* Do the following 16 operations. */
+    SET(a, b, c, d,  1,  5, T17);
+    SET(d, a, b, c,  6,  9, T18);
+    SET(c, d, a, b, 11, 14, T19);
+    SET(b, c, d, a,  0, 20, T20);
+    SET(a, b, c, d,  5,  5, T21);
+    SET(d, a, b, c, 10,  9, T22);
+    SET(c, d, a, b, 15, 14, T23);
+    SET(b, c, d, a,  4, 20, T24);
+    SET(a, b, c, d,  9,  5, T25);
+    SET(d, a, b, c, 14,  9, T26);
+    SET(c, d, a, b,  3, 14, T27);
+    SET(b, c, d, a,  8, 20, T28);
+    SET(a, b, c, d, 13,  5, T29);
+    SET(d, a, b, c,  2,  9, T30);
+    SET(c, d, a, b,  7, 14, T31);
+    SET(b, c, d, a, 12, 20, T32);
+#undef SET
+
+     /* Round 3. */
+     /* Let [abcd k s t] denote the operation
+          a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */
+#define H(x, y, z) ((x) ^ (y) ^ (z))
+#define SET(a, b, c, d, k, s, Ti)\
+  t = a + H(b,c,d) + X[k] + Ti;\
+  a = ROTATE_LEFT(t, s) + b
+     /* Do the following 16 operations. */
+    SET(a, b, c, d,  5,  4, T33);
+    SET(d, a, b, c,  8, 11, T34);
+    SET(c, d, a, b, 11, 16, T35);
+    SET(b, c, d, a, 14, 23, T36);
+    SET(a, b, c, d,  1,  4, T37);
+    SET(d, a, b, c,  4, 11, T38);
+    SET(c, d, a, b,  7, 16, T39);
+    SET(b, c, d, a, 10, 23, T40);
+    SET(a, b, c, d, 13,  4, T41);
+    SET(d, a, b, c,  0, 11, T42);
+    SET(c, d, a, b,  3, 16, T43);
+    SET(b, c, d, a,  6, 23, T44);
+    SET(a, b, c, d,  9,  4, T45);
+    SET(d, a, b, c, 12, 11, T46);
+    SET(c, d, a, b, 15, 16, T47);
+    SET(b, c, d, a,  2, 23, T48);
+#undef SET
+
+     /* Round 4. */
+     /* Let [abcd k s t] denote the operation
+          a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */
+#define I(x, y, z) ((y) ^ ((x) | ~(z)))
+#define SET(a, b, c, d, k, s, Ti)\
+  t = a + I(b,c,d) + X[k] + Ti;\
+  a = ROTATE_LEFT(t, s) + b
+     /* Do the following 16 operations. */
+    SET(a, b, c, d,  0,  6, T49);
+    SET(d, a, b, c,  7, 10, T50);
+    SET(c, d, a, b, 14, 15, T51);
+    SET(b, c, d, a,  5, 21, T52);
+    SET(a, b, c, d, 12,  6, T53);
+    SET(d, a, b, c,  3, 10, T54);
+    SET(c, d, a, b, 10, 15, T55);
+    SET(b, c, d, a,  1, 21, T56);
+    SET(a, b, c, d,  8,  6, T57);
+    SET(d, a, b, c, 15, 10, T58);
+    SET(c, d, a, b,  6, 15, T59);
+    SET(b, c, d, a, 13, 21, T60);
+    SET(a, b, c, d,  4,  6, T61);
+    SET(d, a, b, c, 11, 10, T62);
+    SET(c, d, a, b,  2, 15, T63);
+    SET(b, c, d, a,  9, 21, T64);
+#undef SET
+
+     /* Then perform the following additions. (That is increment each
+        of the four registers by the value it had before this block
+        was started.) */
+    pms->abcd[0] += a;
+    pms->abcd[1] += b;
+    pms->abcd[2] += c;
+    pms->abcd[3] += d;
+}
+
+void
+md5_init(md5_state_t *pms)
+{
+    pms->count[0] = pms->count[1] = 0;
+    pms->abcd[0] = 0x67452301;
+    pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476;
+    pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301;
+    pms->abcd[3] = 0x10325476;
+}
+
+void
+md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes)
+{
+    const md5_byte_t *p = data;
+    int left = nbytes;
+    int offset = (pms->count[0] >> 3) & 63;
+    md5_word_t nbits = (md5_word_t)(nbytes << 3);
+
+    if (nbytes <= 0)
+	return;
+
+    /* Update the message length. */
+    pms->count[1] += nbytes >> 29;
+    pms->count[0] += nbits;
+    if (pms->count[0] < nbits)
+	pms->count[1]++;
+
+    /* Process an initial partial block. */
+    if (offset) {
+	int copy = (offset + nbytes > 64 ? 64 - offset : nbytes);
+
+	memcpy(pms->buf + offset, p, copy);
+	if (offset + copy < 64)
+	    return;
+	p += copy;
+	left -= copy;
+	md5_process(pms, pms->buf);
+    }
+
+    /* Process full blocks. */
+    for (; left >= 64; p += 64, left -= 64)
+	md5_process(pms, p);
+
+    /* Process a final partial block. */
+    if (left)
+	memcpy(pms->buf, p, left);
+}
+
+void
+md5_finish(md5_state_t *pms, md5_byte_t digest[16])
+{
+    static const md5_byte_t pad[64] = {
+	0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+    };
+    md5_byte_t data[8];
+    int i;
+
+    /* Save the length before padding. */
+    for (i = 0; i < 8; ++i)
+	data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3));
+    /* Pad to 56 bytes mod 64. */
+    md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1);
+    /* Append the length. */
+    md5_append(pms, data, 8);
+    for (i = 0; i < 16; ++i)
+	digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3));
+}

data/ext/md5.h

@@ -0,0 +1,101 @@
+/*
+  Copyright (C) 1999, 2002 Aladdin Enterprises.  All rights reserved.
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  L. Peter Deutsch
+  ghost@aladdin.com
+
+ */
+/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */
+/*
+  Independent implementation of MD5 (RFC 1321).
+
+  This code implements the MD5 Algorithm defined in RFC 1321, whose
+  text is available at
+	http://www.ietf.org/rfc/rfc1321.txt
+  The code is derived from the text of the RFC, including the test suite
+  (section A.5) but excluding the rest of Appendix A.  It does not include
+  any code or documentation that is identified in the RFC as being
+  copyrighted.
+
+  The original and principal author of md5.h is L. Peter Deutsch
+  <ghost@aladdin.com>.  Other authors are noted in the change history
+  that follows (in reverse chronological order):
+
+  2002-04-13 lpd Removed support for non-ANSI compilers; removed
+	references to Ghostscript; clarified derivation from RFC 1321;
+	now handles byte order either statically or dynamically.
+  1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
+  1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5);
+	added conditionalization for C++ compilation from Martin
+	Purschke <purschke@bnl.gov>.
+  1999-05-03 lpd Original version.
+ */
+
+#ifndef md5_INCLUDED
+#  define md5_INCLUDED
+
+/*
+ * This package supports both compile-time and run-time determination of CPU
+ * byte order.  If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be
+ * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is
+ * defined as non-zero, the code will be compiled to run only on big-endian
+ * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to
+ * run on either big- or little-endian CPUs, but will run slightly less
+ * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined.
+ */
+
+typedef unsigned char md5_byte_t; /* 8-bit byte */
+typedef unsigned int md5_word_t; /* 32-bit word */
+
+/* Define the state of the MD5 Algorithm. */
+typedef struct md5_state_s {
+    md5_word_t count[2];	/* message length in bits, lsw first */
+    md5_word_t abcd[4];		/* digest buffer */
+    md5_byte_t buf[64];		/* accumulate block */
+} md5_state_t;
+
+#ifdef __cplusplus
+extern "C" 
+{
+#endif
+
+/* Initialize the algorithm. */
+
+#ifdef WIN32
+_declspec(dllexport)
+#endif
+void md5_init(md5_state_t *pms);
+
+/* Append a string to the message. */
+#ifdef WIN32
+_declspec(dllexport)
+#endif
+void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes);
+
+/* Finish the message and return the digest. */
+#ifdef WIN32
+_declspec(dllexport)
+#endif
+void md5_finish(md5_state_t *pms, md5_byte_t digest[16]);
+
+#ifdef __cplusplus
+}  /* end extern "C" */
+#endif
+
+#endif /* md5_INCLUDED */

data/spec/fast_hash_ring_spec.rb

@@ -0,0 +1,185 @@
+require File.join(File.dirname(__FILE__), 'spec_base')
+
+# Basic constants
+UNWEIGHTED_RUNS = 1000
+UNWEIGHTED_ERROR_BOUND = 0.05
+WEIGHTED_RUNS = 1000
+WEIGHTED_ERROR_BOUND = 0.05
+
+describe FastHashRing do
+  include FastHashRingHelpers
+
+  it "should allow the creation of a hash ring with non-string objects" do
+    lambda do
+      ring = FastHashRing.new([FastHashRing.new(['a']), FastHashRing.new(['b'])])
+    end.should_not raise_error
+  end
+
+  describe "bisection" do
+    before do
+      @ring = FastHashRing.new(['a'])
+      @test_array = [10,20,30]
+    end
+
+    it "should return 0 if it less than the first element" do
+      @ring.bisect(@test_array, 5).should eql(0)
+    end
+
+    it "should return the index it should go into to maintain order" do
+      @ring.bisect(@test_array, 15).should eql(1)
+    end
+
+    it "should return the final index if greater than all items" do
+      @ring.bisect(@test_array, 40).should eql(3)
+    end
+  end
+
+  describe "iterating nodes" do
+    before do
+      @ring = FastHashRing.new(['a','b','c'])
+    end
+
+    it "should return correct values based on python" do
+      a_iterate = @ring.iterate_nodes('a')
+      b_iterate = @ring.iterate_nodes('b')
+      c_iterate = @ring.iterate_nodes('ccccccccc')
+
+      a_python = ["a","c","b"]
+      b_python = ["b","c","a"]
+      c_python = ["c","a","b"]
+
+      (a_iterate - a_python).should be_empty
+      (b_iterate - b_python).should be_empty
+      (c_iterate - c_python).should be_empty
+    end
+  end
+
+  describe "getting nodes" do
+    def check_consistent_assigns
+      first_node = @ring.get_node(@consistent_key)
+      
+      100.times do
+        @ring.get_node(@consistent_key).should eql(first_node)
+      end
+    end
+
+    
+    def check_distribution
+      # Keys chosen specifically from trying on Python code
+      first_node = @ring.get_node('a')
+      second_node = @ring.get_node('b')
+      
+      first_node.should_not eql(second_node)
+    end
+
+    def check_probability(run_count, error_bound, weights={})
+      counts = {}
+      total_counts = 0
+      
+      run_count.times do |i|
+        node = @ring.get_node(random_string)
+        
+        if counts[node].nil?
+          counts[node] = 0
+        else
+          counts[node] += 1
+        end
+        
+        total_counts += 1
+      end
+      
+      total_keys = counts.keys.length
+      
+      # Should be bounded, hopefully by 1/total_keys (give or take an error bound)
+      ideal_probability = (1.0/total_keys) + error_bound
+      counts.each do |node, count|
+        weight = weights[node] || 1
+        probability = (count / run_count.to_f)
+        weighted_probability = ideal_probability * weight
+
+        if probability >= weighted_probability
+          fail "#{node} has probability: #{probability}"
+        end
+      end
+    end
+
+    describe "without explicit weights" do
+      before do
+        @ring = FastHashRing.new(['a','b','c'])
+        @consistent_key = 'Hello, World'
+      end
+
+      it "should consistently assign nodes" do
+        check_consistent_assigns
+      end
+
+      it "should distribute keys to different buckets" do
+        check_distribution
+      end
+
+      it "should assign keys fairly randomly" do
+        check_probability(UNWEIGHTED_RUNS, UNWEIGHTED_ERROR_BOUND)
+      end
+    end
+
+    describe "with explicit weights" do
+      before do
+        # Create a hash ring with 'a' having a 2:1 weight
+        @weights = { 'a' => 2 }
+        @ring = FastHashRing.new(['a','b','c'], @weights)
+        @consistent_key = 'Hello, World'
+      end
+
+      it "should consistently assign nodes" do
+        check_consistent_assigns
+      end
+
+      it "should distribute keys to different buckets" do
+        check_distribution
+      end
+
+      it "should assign keys fairly randomly, but according to weights" do
+        check_probability(WEIGHTED_RUNS, WEIGHTED_ERROR_BOUND, @weights)
+      end
+    end
+  end
+
+  describe "hashing methods" do
+    before do
+      @ring = FastHashRing.new(['a'])
+    end
+
+    it "should return the raw digest for _hash_digest" do
+      random_string = 'some random string'
+
+      m = Digest::MD5.new
+      m.update(random_string)
+      
+      @ring._hash_digest(random_string).should eql(m.digest)
+    end
+
+    it "should match the python output for _hash_val" do
+      # This output was taken directly from the python library
+      py_output = 2830561728
+      ruby_output = @ring._hash_val(@ring._hash_digest('a')) { |x| x+4 }
+
+      ruby_output.should eql(py_output)
+    end
+  end
+
+  # THIS IS A VERY DIRTY WAY TO SPEC THIS
+  # But given its "random" nature, I figured comparing the two libraries'
+  # (one of which is in production on a huge site) output should be
+  # "safe enough"
+  describe "ring generation" do
+    it "should generate the same ring as python, given the same inputs" do
+      # Yeah... I know... terrible.
+      py_output = [3747649, 3747649, 35374473, 35374473, 61840307, 61840307, 82169324, 82169324, 99513906, 99513906, 171267966, 171267966, 189092589, 189092589, 211562723, 211562723, 274168570, 274168570, 309884358, 309884358, 337859634, 337859634, 359487305, 359487305, 437877875, 437877875, 440532511, 440532511, 441427647, 441427647, 540691923, 540691923, 561744136, 561744136, 566640950, 566640950, 573631360, 573631360, 593354384, 593354384, 616375601, 616375601, 653401705, 653401705, 658933707, 658933707, 711407824, 711407824, 717967565, 717967565, 791654246, 791654246, 815230777, 815230777, 836319689, 836319689, 943387296, 943387296, 948212432, 948212432, 954761114, 954761114, 983151602, 983151602, 1041951938, 1041951938, 1044903177, 1044903177, 1109542669, 1109542669, 1215807553, 1215807553, 1234529376, 1234529376, 1240978794, 1240978794, 1241570279, 1241570279, 1245440929, 1245440929, 1295496069, 1295496069, 1359345465, 1359345465, 1371916815, 1371916815, 1440228341, 1440228341, 1463589668, 1463589668, 1542595588, 1542595588, 1571041323, 1571041323, 1580821462, 1580821462, 1609040193, 1609040193, 1663806909, 1663806909, 1673418579, 1673418579, 1725587406, 1725587406, 1743807106, 1743807106, 1745454947, 1745454947, 1770079607, 1770079607, 1816647406, 1816647406, 1823214399, 1823214399, 1858099396, 1858099396, 1889941457, 1889941457, 1903777629, 1903777629, 1956489818, 1956489818, 1981836821, 1981836821, 2027012493, 2027012493, 2036573472, 2036573472, 2063971870, 2063971870, 2113406442, 2113406442, 2203084188, 2203084188, 2245550483, 2245550483, 2369128516, 2369128516, 2401481896, 2401481896, 2405232024, 2405232024, 2439876819, 2439876819, 2498655628, 2498655628, 2666618195, 2666618195, 2709250454, 2709250454, 2725462545, 2725462545, 2761971368, 2761971368, 2820158560, 2820158560, 2847935782, 2847935782, 2873909817, 2873909817, 2960677255, 2960677255, 2970346521, 2970346521, 3065786853, 3065786853, 3173507458, 3173507458, 3187067483, 3187067483, 3189484171, 3189484171, 3196179889, 3196179889, 3200322582, 3200322582, 3234564840, 3234564840, 3262283799, 3262283799, 3310202261, 3310202261, 3326019031, 3326019031, 3332298302, 3332298302, 3347538539, 3347538539, 3365852132, 3365852132, 3378546819, 3378546819, 3430078214, 3430078214, 3453809654, 3453809654, 3467283568, 3467283568, 3469681976, 3469681976, 3494401641, 3494401641, 3522127265, 3522127265, 3523123410, 3523123410, 3555788439, 3555788439, 3585259232, 3585259232, 3587218875, 3587218875, 3587230532, 3587230532, 3627100732, 3627100732, 3642352831, 3642352831, 3670553958, 3670553958, 3721827301, 3721827301, 3746479890, 3746479890, 3836178086, 3836178086, 3887780209, 3887780209, 3927215372, 3927215372, 3953297430, 3953297430, 3967308270, 3967308270, 4025490138, 4025490138, 4045625605, 4045625605, 4094112530, 4094112530]
+
+      ruby_output = FastHashRing.new(['a'])
+
+      # Calculate the difference of the array, since ordering may be different
+      (ruby_output.sorted_keys - py_output).should be_empty
+    end
+  end
+end

data/spec/spec_base.rb

@@ -0,0 +1,9 @@
+# Include the hash_ring library
+require 'fast_hash_ring'
+
+# Helpers
+module FastHashRingHelpers
+  def random_string(length=50)
+    (0...length).map{ ('a'..'z').to_a[rand(26)] }.join
+  end
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: fast_hash_ring
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- Flinn
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-15 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: 
+email: flinn@actsasflinn.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- CREDITS
+- LICENSE
+- Rakefile
+- README.rdoc
+- ext/fast_hash_ring.c
+- ext/fast_hash_ring.h
+- ext/md5.c
+- ext/md5.h
+- spec/fast_hash_ring_spec.rb
+- spec/spec_base.rb
+- benchmarks/fast_hash_ring.rb
+has_rdoc: true
+homepage: http://github.com/actsasflinn/fast_hash_ring/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A Ruby C Extension based on the pure Ruby HashRing gem
+test_files: 
+- spec/spec_base.rb