mitchellh-hash_ring 0.1

data/CREDITS

@@ -0,0 +1,9 @@
+======================================
+= hash_ring credits                  =
+======================================
+
+Original Author (Python Implementation):
+  Amir Salihefendic http://amix.dk
+
+Ported to Ruby by:
+  Mitchell Hashimoto http://mitchellhashimoto.com

data/LICENSE

@@ -0,0 +1,10 @@
+Copyright (c) 2009, Mitchell Hashimoto
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    * Neither the name of the owner nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.rdoc

@@ -0,0 +1,24 @@
+= hash_ring
+
+== Background
+
+hash_ring is a pure Ruby implementation of consistent hashing. 
+hash_ring is based on the original Python code written by
+Amir Salihefendic. A comprehensive blog post detailing the methods
+and reasoning for such a library can be viewed by visiting the following
+URL:
+
+http://amix.dk/blog/viewEntry/19367
+
+== Usage
+
+  memcache_servers = ['192.168.0.111:14107',
+                      '192.168.0.112:14107',
+                      '192.168.0.113:14108']
+ 
+  # Since server 1 has double the RAM, lets weight it
+  # twice as much to get twice the keys. This is optional
+  weights = { '192.168.0.111' => 2 }
+
+  ring = HashRing.new(memcache_servers, weights)
+  server = ring.get_node('my_key')

data/Rakefile

@@ -0,0 +1,92 @@
+require 'rake'
+require 'rake/clean'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'spec/rake/spectask'
+require 'fileutils'
+
+load 'hash_ring.gemspec'
+
+###################################
+# Clean & Defaut Task
+###################################
+CLEAN.include('dist','tmp','rdoc')
+task :default => [:clean, :repackage]
+
+###################################
+# Specs
+################################### 
+desc "Run all specs for hash_ring"
+Spec::Rake::SpecTask.new('spec') do |t|
+  t.spec_files = FileList['spec/**/*.rb']
+end
+
+###################################
+# Docs
+###################################
+Rake::RDocTask.new do |rd|
+  rd.main = 'README.rdoc'
+ 
+  rd.rdoc_dir = 'doc'
+ 
+  rd.rdoc_files.include(
+    'README.rdoc',
+    'LICENSE',
+    'CREDITS',
+    'lib/**/*.rb')
+ 
+  rd.title = 'hash_ring'
+ 
+  rd.options << '-N' # line numbers
+  rd.options << '-S' # inline source
+end
+
+###################################
+# Packaging - Thank you Sinatra
+################################### 
+# Load the gemspec using the same limitations as github
+def spec
+  @spec ||=
+    begin
+      require 'rubygems/specification'
+      data = File.read('hash_ring.gemspec')
+      spec = nil
+      Thread.new { spec = eval("$SAFE = 3\n#{data}") }.join
+      spec
+    end
+end
+ 
+def package(ext='')
+  "dist/hash_ring-#{spec.version}" + ext
+end
+ 
+desc 'Build packages'
+task :package => %w[.gem .tar.gz].map {|e| package(e)}
+ 
+desc 'Build and install as local gem'
+task :install => package('.gem') do
+  sh "gem install #{package('.gem')}"
+end
+ 
+directory 'dist/'
+CLOBBER.include('dist')
+ 
+file package('.gem') => %w[dist/ hash_ring.gemspec] + spec.files do |f|
+  sh "gem build hash_ring.gemspec"
+  mv File.basename(f.name), f.name
+end
+ 
+file package('.tar.gz') => %w[dist/] + spec.files do |f|
+  sh <<-SH
+git archive \
+--prefix=hash_ring-#{source_version}/ \
+--format=tar \
+HEAD | gzip > #{f.name}
+SH
+end
+
+def source_version
+  line = File.read('lib/hash_ring.rb')[/^\s*VERSION = .*/]
+  line.match(/.*VERSION = '(.*)'/)[1]
+end

data/lib/hash_ring.rb

@@ -0,0 +1,187 @@
+######################################
+# hash_ring
+# Code ported from Python version written by Amir Salihefendic
+######################################
+# Copyright (c) 2009, Mitchell Hashimoto, mitchell.hashimoto@gmail.com
+#
+
+require 'digest/md5'
+
+# = HashRing Class
+#
+# == Background
+#
+# Implements consistent hashing that can be used when
+# the number of server nodes can increase or decrease (like in memcached).
+#
+# Consistent hashing is a scheme that provides a hash table functionality
+# in a way that the adding or removing of one slot
+# does not significantly change the mapping of keys to slots.
+#
+# More information about consistent hashing can be read in these articles:
+#
+# "Web Caching with Consistent Hashing":
+#    http://www8.org/w8-papers/2a-webserver/caching/paper2.html
+#
+# "Consistent hashing and random trees:
+#    Distributed caching protocols for relieving hot spots on the World Wide Web (1997)":
+#    http://citeseerx.ist.psu.edu/legacymapper?did=38148
+#
+# == Usage
+#
+#  memcache_servers = ['192.168.0.111:14107',
+#                      '192.168.0.112:14107',
+#                      '192.168.0.113:14108']
+#
+#  # Since server 1 has double the RAM, lets weight it
+#  # twice as much to get twice the keys. This is optional
+#  weights = { '192.168.0.111' => 2 }
+#
+#  ring = HashRing.new(memcache_servers, weights)
+#  server = ring.get_node('my_key')
+#
+class HashRing
+  VERSION = '0.1'
+
+  #
+  # Creates a HashRing instance
+  #
+  # == parameters
+  #
+  #   * nodes - A list of objects which have a proper to_s representation.
+  #   * weights - A hash (dictionary, not to be mixed up with HashRing)
+  #       which sets weights to the nodes. The default weight is that all
+  #       nodes have equal weight.
+  def initialize(nodes=nil, weights=nil)
+    @ring = {}
+    @_sorted_keys = []
+
+    @nodes = nodes
+
+    weights = {} if weights.nil?
+    
+    @weights = weights
+
+    self._generate_circle()
+    self
+  end
+
+  #
+  # Generates the circle
+  def _generate_circle
+    total_weight = 0
+    
+    @nodes.each do |node|
+      total_weight += @weights[node] || 1
+    end
+
+    @nodes.each do |node|
+      weight = @weights[node] || 1
+      factor = ((40 * @nodes.length * weight) / total_weight.to_f).floor.to_i
+
+      factor.times do |j|
+        b_key = self._hash_digest("#{node}-#{j}")
+
+        3.times do |i|
+          key = self._hash_val(b_key) { |x| x+(i*4) }
+          @ring[key] = node
+          @_sorted_keys.push(key)
+        end
+      end
+    end
+
+    @_sorted_keys.sort!
+  end
+
+  #
+  # Given a string key a corresponding node is returned. If the
+  # ring is empty, nil is returned.
+  def get_node(string_key)
+    pos = self.get_node_pos(string_key)
+    return nil if pos.nil?
+
+    return @ring[@_sorted_keys[pos]]
+  end
+
+  #
+  # Given a string key a corresponding node's position in the ring
+  # is returned. Nil is returned if the ring is empty.
+  def get_node_pos(string_key)
+    return nil if @ring.empty?
+
+    key = self.gen_key(string_key)
+    nodes = @_sorted_keys
+    pos = bisect(nodes, key)
+
+    if pos == nodes.length
+      return 0
+    else
+      return pos
+    end
+  end
+
+  #
+  # Returns an array of nodes where the key could be stored, starting
+  # at the correct position.
+  def iterate_nodes(string_key)
+    returned_values = []
+    pos = self.get_node_pos(string_key)
+    @_sorted_keys[pos, @_sorted_keys.length].each do |ring_index|
+      key = @ring[ring_index]
+      next if returned_values.include?(key)
+      returned_values.push(key)
+    end
+    
+    @_sorted_keys.each_index do |i|
+      break if i >= pos
+
+      key = @ring[@_sorted_keys[i]]
+      next if returned_values.include?(key)
+      returned_values.push(key)      
+    end
+
+    returned_values
+  end
+
+  #
+  # Given a string key this returns a long value. This long value
+  # represents a location on the ring.
+  #
+  # MD5 is used currently.
+  def gen_key(string_key)
+    b_key = self._hash_digest(string_key)
+    return self._hash_val(b_key) { |x| x }
+  end
+
+  def _hash_val(b_key, &block)
+    return ((b_key[block.call(3)] << 24) | 
+            (b_key[block.call(2)] << 16) | 
+            (b_key[block.call(1)] << 8) | 
+            (b_key[block.call(0)])) 
+ end
+
+  #
+  # Returns raw MD5 digest given a key
+  def _hash_digest(key)
+    m = Digest::MD5.new
+    m.update(key)
+
+    # No need to ord each item since ordinary array access
+    # of a string in Ruby converts to ordinal value
+    return m.digest
+  end
+
+  #
+  # Bisect an array
+  def bisect(arr, key)
+    arr.each_index do |i|
+      return i if key < arr[i]
+    end
+
+    return arr.length
+  end
+  
+  def sorted_keys #:nodoc:
+    @_sorted_keys
+  end
+end

data/spec/hash_ring_spec.rb

@@ -0,0 +1,179 @@
+require File.join(File.dirname(__FILE__), 'spec_base')
+
+# Basic constants
+UNWEIGHTED_RUNS = 1000
+UNWEIGHTED_ERROR_BOUND = 0.05
+WEIGHTED_RUNS = 1000
+WEIGHTED_ERROR_BOUND = 0.05
+
+describe HashRing do
+  include HashRingHelpers
+
+  describe "bisection" do
+    before do
+      @ring = HashRing.new(['a'])
+      @test_array = [10,20,30]
+    end
+
+    it "should return 0 if it less than the first element" do
+      @ring.bisect(@test_array, 5).should eql(0)
+    end
+
+    it "should return the index it should go into to maintain order" do
+      @ring.bisect(@test_array, 15).should eql(1)
+    end
+
+    it "should return the final index if greater than all items" do
+      @ring.bisect(@test_array, 40).should eql(3)
+    end
+  end
+
+  describe "iterating nodes" do
+    before do
+      @ring = HashRing.new(['a','b','c'])
+    end
+
+    it "should return correct values based on python" do
+      a_iterate = @ring.iterate_nodes('a')
+      b_iterate = @ring.iterate_nodes('b')
+      c_iterate = @ring.iterate_nodes('ccccccccc')
+
+      a_python = ["a","c","b"]
+      b_python = ["b","c","a"]
+      c_python = ["c","a","b"]
+
+      (a_iterate - a_python).should be_empty
+      (b_iterate - b_python).should be_empty
+      (c_iterate - c_python).should be_empty
+    end
+  end
+
+  describe "getting nodes" do
+    def check_consistent_assigns
+      first_node = @ring.get_node(@consistent_key)
+      
+      100.times do
+        @ring.get_node(@consistent_key).should eql(first_node)
+      end
+    end
+
+    
+    def check_distribution
+      # Keys chosen specifically from trying on Python code
+      first_node = @ring.get_node('a')
+      second_node = @ring.get_node('b')
+      
+      first_node.should_not eql(second_node)
+    end
+
+    def check_probability(run_count, error_bound, weights={})
+      counts = {}
+      total_counts = 0
+      
+      run_count.times do |i|
+        node = @ring.get_node(random_string)
+        
+        if counts[node].nil?
+          counts[node] = 0
+        else
+          counts[node] += 1
+        end
+        
+        total_counts += 1
+      end
+      
+      total_keys = counts.keys.length
+      
+      # Should be bounded, hopefully by 1/total_keys (give or take an error bound)
+      ideal_probability = (1.0/total_keys) + error_bound
+      counts.each do |node, count|
+        weight = weights[node] || 1
+        probability = (count / run_count.to_f)
+        weighted_probability = ideal_probability * weight
+
+        if probability >= weighted_probability
+          fail "#{node} has probability: #{probability}"
+        end
+      end
+    end
+
+    describe "without explicit weights" do
+      before do
+        @ring = HashRing.new(['a','b','c'])
+        @consistent_key = 'Hello, World'
+      end
+
+      it "should consistently assign nodes" do
+        check_consistent_assigns
+      end
+
+      it "should distribute keys to different buckets" do
+        check_distribution
+      end
+
+      it "should assign keys fairly randomly" do
+        check_probability(UNWEIGHTED_RUNS, UNWEIGHTED_ERROR_BOUND)
+      end
+    end
+
+    describe "with explicit weights" do
+      before do
+        # Create a hash ring with 'a' having a 2:1 weight
+        @weights = { 'a' => 2 }
+        @ring = HashRing.new(['a','b','c'], @weights)
+        @consistent_key = 'Hello, World'
+      end
+
+      it "should consistently assign nodes" do
+        check_consistent_assigns
+      end
+
+      it "should distribute keys to different buckets" do
+        check_distribution
+      end
+
+      it "should assign keys fairly randomly, but according to weights" do
+        check_probability(WEIGHTED_RUNS, WEIGHTED_ERROR_BOUND, @weights)
+      end
+    end
+  end
+
+  describe "hashing methods" do
+    before do
+      @ring = HashRing.new(['a'])
+    end
+
+    it "should return the raw digest for _hash_digest" do
+      random_string = 'some random string'
+
+      m = Digest::MD5.new
+      m.update(random_string)
+      
+      @ring._hash_digest(random_string).should eql(m.digest)
+    end
+
+    it "should match the python output for _hash_val" do
+      # This output was taken directly from the python library
+      py_output = 2830561728
+      ruby_output = @ring._hash_val(@ring._hash_digest('a')) { |x| x+4 }
+
+      ruby_output.should eql(py_output)
+    end
+  end
+
+  # THIS IS A VERY DIRTY WAY TO SPEC THIS
+  # But given its "random" nature, I figured comparing the two libraries'
+  # (one of which is in production on a huge site) output should be
+  # "safe enough"
+  describe "ring generation" do
+    it "should generate the same ring as python, given the same inputs" do
+      # Yeah... I know... terrible.
+      py_output = [3747649, 3747649, 35374473, 35374473, 61840307, 61840307, 82169324, 82169324, 99513906, 99513906, 171267966, 171267966, 189092589, 189092589, 211562723, 211562723, 274168570, 274168570, 309884358, 309884358, 337859634, 337859634, 359487305, 359487305, 437877875, 437877875, 440532511, 440532511, 441427647, 441427647, 540691923, 540691923, 561744136, 561744136, 566640950, 566640950, 573631360, 573631360, 593354384, 593354384, 616375601, 616375601, 653401705, 653401705, 658933707, 658933707, 711407824, 711407824, 717967565, 717967565, 791654246, 791654246, 815230777, 815230777, 836319689, 836319689, 943387296, 943387296, 948212432, 948212432, 954761114, 954761114, 983151602, 983151602, 1041951938, 1041951938, 1044903177, 1044903177, 1109542669, 1109542669, 1215807553, 1215807553, 1234529376, 1234529376, 1240978794, 1240978794, 1241570279, 1241570279, 1245440929, 1245440929, 1295496069, 1295496069, 1359345465, 1359345465, 1371916815, 1371916815, 1440228341, 1440228341, 1463589668, 1463589668, 1542595588, 1542595588, 1571041323, 1571041323, 1580821462, 1580821462, 1609040193, 1609040193, 1663806909, 1663806909, 1673418579, 1673418579, 1725587406, 1725587406, 1743807106, 1743807106, 1745454947, 1745454947, 1770079607, 1770079607, 1816647406, 1816647406, 1823214399, 1823214399, 1858099396, 1858099396, 1889941457, 1889941457, 1903777629, 1903777629, 1956489818, 1956489818, 1981836821, 1981836821, 2027012493, 2027012493, 2036573472, 2036573472, 2063971870, 2063971870, 2113406442, 2113406442, 2203084188, 2203084188, 2245550483, 2245550483, 2369128516, 2369128516, 2401481896, 2401481896, 2405232024, 2405232024, 2439876819, 2439876819, 2498655628, 2498655628, 2666618195, 2666618195, 2709250454, 2709250454, 2725462545, 2725462545, 2761971368, 2761971368, 2820158560, 2820158560, 2847935782, 2847935782, 2873909817, 2873909817, 2960677255, 2960677255, 2970346521, 2970346521, 3065786853, 3065786853, 3173507458, 3173507458, 3187067483, 3187067483, 3189484171, 3189484171, 3196179889, 3196179889, 3200322582, 3200322582, 3234564840, 3234564840, 3262283799, 3262283799, 3310202261, 3310202261, 3326019031, 3326019031, 3332298302, 3332298302, 3347538539, 3347538539, 3365852132, 3365852132, 3378546819, 3378546819, 3430078214, 3430078214, 3453809654, 3453809654, 3467283568, 3467283568, 3469681976, 3469681976, 3494401641, 3494401641, 3522127265, 3522127265, 3523123410, 3523123410, 3555788439, 3555788439, 3585259232, 3585259232, 3587218875, 3587218875, 3587230532, 3587230532, 3627100732, 3627100732, 3642352831, 3642352831, 3670553958, 3670553958, 3721827301, 3721827301, 3746479890, 3746479890, 3836178086, 3836178086, 3887780209, 3887780209, 3927215372, 3927215372, 3953297430, 3953297430, 3967308270, 3967308270, 4025490138, 4025490138, 4045625605, 4045625605, 4094112530, 4094112530]
+
+      ruby_output = HashRing.new(['a'])
+
+      # Calculate the difference of the array, since ordering may be different
+      (ruby_output.sorted_keys - py_output).should be_empty
+    end
+  end
+end

data/spec/spec_base.rb

@@ -0,0 +1,9 @@
+# Include the hash_ring library
+require File.expand_path(File.dirname(__FILE__) + '/../lib/hash_ring')
+
+# Helpers
+module HashRingHelpers
+  def random_string(length=50)
+    (0...length).map{ ('a'..'z').to_a[rand(26)] }.join
+  end
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: mitchellh-hash_ring
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- Mitchell Hashimoto
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-04 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: hash_ring implementation in Ruby
+email: mitchell.hashimoto@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- CREDITS
+- LICENSE
+- README.rdoc
+- Rakefile
+- lib/hash_ring.rb
+- spec/spec_base.rb
+- spec/hash_ring_spec.rb
+has_rdoc: true
+homepage: http://github.com/mitchellh/hash_ring/
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- hash_ring
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Consistent hashing implemented in Ruby
+test_files: []
+