jmhodges-murmur_hash 1.0.0

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2009-05-02
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,8 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/murmur_hash.rb
+test/test_murmur_hash.rb
+ext/murmur/extconf.rb
+ext/murmur/murmur_hash.cpp

data/README.txt

@@ -0,0 +1,68 @@
+= murmur_hash
+
+* http://github.com/jmhodges/murmur_hash
+
+== DESCRIPTION:
+
+* A ruby extension for the MurmurHash2 functions. See http://murmurhash.googlepages.com/
+
+== FEATURES/PROBLEMS:
+
+* Provides all 3 versions of the 32-bit MurmurHash functions which
+  are:
+  
+  * `MurmurHash.murmur_hash`, the simplest and fastest version of
+    MurmurHash2. Note: output depends on the endianness of the machine
+    it is run on.
+  * `MurmurHash.neutral_murmur_hash`, a version of the
+    straight-forward MurmurHash2 function that does not depend on the
+    endianness of the machine it is run on. Slightly slower.
+  * `MurmurHash.aligned_murmur_hash`, a version of the MurmurHash2
+    function that does only byte-aligned reads. It is slightly safer
+    on some uncommon platforms.
+
+* Provides the 64-bit MurmurHash function. When on a 32-bit machine,
+  it will use the backward compatible version.
+
+== SYNOPSIS:
+  # 23 can be any unsigned 32-bit integer (i.e. from 0 to 2**32 - 1)
+  hash_number = MurmurHash.murmur_hash("somestring", 23) 
+
+  h = MurmurHash.neutral_murmur_hash("s", 23)
+  h = MurmurHash.aligned_murmur_hash("s", 23)
+
+  # 23 still must be a unsigned 32-bit integer
+  h = MurmurHash.murmur_hash64("s", 23)
+
+== REQUIREMENTS:
+
+* rake-compiler for compilation
+
+== INSTALL:
+
+* sudo gem install murmur_hash
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Jeff Hodges
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,84 @@
+# -*- ruby -*-
+require 'rubygems'
+require 'rake'
+
+def define_rake_stuff
+  require 'hoe'
+  begin
+    require './lib/murmur_hash.rb'
+
+  rescue LoadError => e
+    murmur_bundle = File.expand_path(File.dirname(__FILE__) + '/lib/murmur')
+    if (e.message =~ /#{murmur_bundle}(\.bundle)?$/)
+      STDERR.puts "Looks like you have not run a successful `rake compile`, yet. Do that next!"
+    else
+      raise e
+    end
+  end
+  Rake::ExtensionTask.new('murmur')
+  define_hoe_tasks
+end
+
+def define_hoe_tasks
+  version = '0'
+  if defined?(MurmurHash)
+    version = MurmurHash::VERSION
+  end
+
+  h = Hoe.new('murmur_hash', version) do |p|
+    # p.rubyforge_name = 'murmur_hash' # if different than lowercase project name
+    p.developer('Jeff Hodges', 'jeff@somethingsimilar.com')
+    p.extra_deps << ['rake-compiler', '~>0.5']
+  end
+  
+  Rake::Task[:test].prerequisites << :compile
+  h
+end
+
+begin
+  require 'rake/extensiontask'
+  define_rake_stuff
+rescue LoadError => e
+  STDERR.puts(
+              "Be sure to run 'sudo rake setup' because you don't have all the gems installed"
+              )
+end
+
+# Putting this out here cuts off a great deal of time.
+@gems = Gem::SourceIndex.from_installed_gems
+
+def conditionally_install_gem(name, version_requirements)
+
+  di = Gem::DependencyInstaller.new
+  hoe_dep = Gem::Dependency.new('hoe', "~> 1.12")
+  if @gems.search(hoe_dep).empty?
+    di.install hoe_dep
+  end
+end
+
+task :default => :compile
+
+# Taken liberally from http://blog.labnotes.org/2008/02/28/svn-checkout-rake-setup/
+desc "If you're building from source, run this task first to setup the necessary dependencies."
+task :setup do
+  require 'rubygems'
+  require 'rubygems/dependency_installer'
+  require 'rubygems/requirement'
+
+  puts "Checking for gems that need to be installed."
+  conditionally_install_gem('hoe', '~>1.12')
+
+  require 'hoe'
+  h = define_hoe_tasks
+
+  # Runtime dependencies from the Gem's spec
+  dependencies = h.spec.dependencies
+
+  dependencies.each do |dep|
+    conditionally_install_gem(dep.name, dep.version_requirements)
+  end
+
+  puts "\nAnd done."
+end
+
+# vim: syntax=Ruby

data/ext/murmur/extconf.rb

@@ -0,0 +1,6 @@
+require 'mkmf'
+
+dir_config("murmur")
+have_library("c", "main")
+
+create_makefile("murmur")

data/ext/murmur/murmur_hash.cpp

@@ -0,0 +1,429 @@
+//-----------------------------------------------------------------------------
+// MurmurHash2, by Austin Appleby
+
+// Note - This code makes a few assumptions about how your machine behaves -
+
+// 1. We can read a 4-byte value from any address without crashing
+// 2. sizeof(int) == 4
+
+// And it has a few limitations -
+
+// 1. It will not work incrementally.
+// 2. It will not produce the same results on little-endian and big-endian
+//    machines.
+#include <stdio.h>
+#include <stdint.h>
+#include "ruby.h"
+#include "limits.h"
+
+unsigned int MurmurHash2 ( const void * key, int len, unsigned int seed )
+{
+  // 'm' and 'r' are mixing constants generated offline.
+  // They're not really 'magic', they just happen to work well.
+
+  const unsigned int m = 0x5bd1e995;
+  const int r = 24;
+
+  // Initialize the hash to a 'random' value
+
+  unsigned int h = seed ^ len;
+
+  // Mix 4 bytes at a time into the hash
+
+  const unsigned char * data = (const unsigned char *)key;
+
+  while(len >= 4)
+    {
+      unsigned int k = *(unsigned int *)data;
+
+      k *= m;
+      k ^= k >> r;
+      k *= m;
+
+      h *= m;
+      h ^= k;
+
+      data += 4;
+      len -= 4;
+    }
+
+  // Handle the last few bytes of the input array
+
+  switch(len)
+    {
+    case 3: h ^= data[2] << 16;
+    case 2: h ^= data[1] << 8;
+    case 1: h ^= data[0];
+      h *= m;
+    };
+
+  // Do a few final mixes of the hash to ensure the last few
+  // bytes are well-incorporated.
+
+  h ^= h >> 13;
+  h *= m;
+  h ^= h >> 15;
+
+  return h;
+}
+
+//-----------------------------------------------------------------------------
+// MurmurHashAligned2, by Austin Appleby
+
+// Same algorithm as MurmurHash2, but only does aligned reads - should be safer
+// on certain platforms.
+
+// Performance will be lower than MurmurHash2
+
+#define MURMUR_MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; }
+
+unsigned int MurmurHashAligned2 ( const void * key, int len, unsigned int seed )
+{
+  const unsigned int m = 0x5bd1e995;
+  const int r = 24;
+
+  const unsigned char * data = (const unsigned char *)key;
+
+  unsigned int h = seed ^ len;
+
+  int align = (int)data & 3;
+
+  if(align && (len >= 4))
+    {
+      // Pre-load the temp registers
+
+      unsigned int t = 0, d = 0;
+
+      switch(align)
+        {
+        case 1: t |= data[2] << 16;
+        case 2: t |= data[1] << 8;
+        case 3: t |= data[0];
+        }
+
+      t <<= (8 * align);
+
+      data += 4-align;
+      len -= 4-align;
+
+      int sl = 8 * (4-align);
+      int sr = 8 * align;
+
+      // Mix
+
+      while(len >= 4)
+        {
+          d = *(unsigned int *)data;
+          t = (t >> sr) | (d << sl);
+
+          unsigned int k = t;
+
+          MURMUR_MIX(h,k,m);
+
+          t = d;
+
+          data += 4;
+          len -= 4;
+        }
+
+      // Handle leftover data in temp registers
+
+      d = 0;
+
+      if(len >= align)
+        {
+          switch(align)
+            {
+            case 3: d |= data[2] << 16;
+            case 2: d |= data[1] << 8;
+            case 1: d |= data[0];
+            }
+
+          unsigned int k = (t >> sr) | (d << sl);
+          MURMUR_MIX(h,k,m);
+
+          data += align;
+          len -= align;
+
+          //----------
+          // Handle tail bytes
+
+          switch(len)
+            {
+            case 3: h ^= data[2] << 16;
+            case 2: h ^= data[1] << 8;
+            case 1: h ^= data[0];
+              h *= m;
+            };
+        }
+      else
+        {
+          switch(len)
+            {
+            case 3: d |= data[2] << 16;
+            case 2: d |= data[1] << 8;
+            case 1: d |= data[0];
+            case 0: h ^= (t >> sr) | (d << sl);
+              h *= m;
+            }
+        }
+
+      h ^= h >> 13;
+      h *= m;
+      h ^= h >> 15;
+
+      return h;
+    }
+  else
+    {
+      while(len >= 4)
+        {
+          unsigned int k = *(unsigned int *)data;
+
+          MURMUR_MIX(h,k,m);
+
+          data += 4;
+          len -= 4;
+        }
+
+      //----------
+      // Handle tail bytes
+
+      switch(len)
+        {
+        case 3: h ^= data[2] << 16;
+        case 2: h ^= data[1] << 8;
+        case 1: h ^= data[0];
+          h *= m;
+        };
+
+      h ^= h >> 13;
+      h *= m;
+      h ^= h >> 15;
+
+      return h;
+    }
+}
+
+//-----------------------------------------------------------------------------
+// MurmurHashNeutral2, by Austin Appleby
+
+// Same as MurmurHash2, but endian- and alignment-neutral.
+// Half the speed though, alas.
+
+unsigned int MurmurHashNeutral2 ( const void * key, int len, unsigned int seed )
+{
+  const unsigned int m = 0x5bd1e995;
+  const int r = 24;
+
+  unsigned int h = seed ^ len;
+
+  const unsigned char * data = (const unsigned char *)key;
+
+  while(len >= 4)
+    {
+      unsigned int k;
+
+      k  = data[0];
+      k |= data[1] << 8;
+      k |= data[2] << 16;
+      k |= data[3] << 24;
+
+      k *= m;
+      k ^= k >> r;
+      k *= m;
+
+      h *= m;
+      h ^= k;
+
+      data += 4;
+      len -= 4;
+    }
+
+  switch(len)
+    {
+    case 3: h ^= data[2] << 16;
+    case 2: h ^= data[1] << 8;
+    case 1: h ^= data[0];
+      h *= m;
+    };
+
+  h ^= h >> 13;
+  h *= m;
+  h ^= h >> 15;
+
+  return h;
+}
+
+//-----------------------------------------------------------------------------
+// MurmurHash2, 64-bit versions, by Austin Appleby
+
+// The same caveats as 32-bit MurmurHash2 apply here - beware of alignment
+// and endian-ness issues if used across multiple platforms.
+
+// 64-bit hash for 64-bit platforms
+
+uint64_t MurmurHash64A ( const void * key, int len, unsigned int seed )
+{
+	const uint64_t m = 0xc6a4a7935bd1e995LL;
+	const int r = 47;
+
+	uint64_t h = seed ^ (len * m);
+
+	const uint64_t * data = (const uint64_t *)key;
+	const uint64_t * end = data + (len/8);
+
+	while(data != end)
+	{
+		uint64_t k = *data++;
+
+		k *= m;
+		k ^= k >> r;
+		k *= m;
+
+		h ^= k;
+		h *= m;
+	}
+
+	const unsigned char * data2 = (const unsigned char*)data;
+
+	switch(len & 7)
+	{
+	case 7: h ^= uint64_t(data2[6]) << 48;
+	case 6: h ^= uint64_t(data2[5]) << 40;
+	case 5: h ^= uint64_t(data2[4]) << 32;
+	case 4: h ^= uint64_t(data2[3]) << 24;
+	case 3: h ^= uint64_t(data2[2]) << 16;
+	case 2: h ^= uint64_t(data2[1]) << 8;
+	case 1: h ^= uint64_t(data2[0]);
+	        h *= m;
+	};
+
+	h ^= h >> r;
+	h *= m;
+	h ^= h >> r;
+
+	return h;
+}
+
+
+// 64-bit hash for 32-bit platforms
+
+uint64_t MurmurHash64B ( const void * key, int len, unsigned int seed )
+{
+	const unsigned int m = 0x5bd1e995;
+	const int r = 24;
+
+	unsigned int h1 = seed ^ len;
+	unsigned int h2 = 0;
+
+	const unsigned int * data = (const unsigned int *)key;
+
+	while(len >= 8)
+	{
+		unsigned int k1 = *data++;
+		k1 *= m; k1 ^= k1 >> r; k1 *= m;
+		h1 *= m; h1 ^= k1;
+		len -= 4;
+
+		unsigned int k2 = *data++;
+		k2 *= m; k2 ^= k2 >> r; k2 *= m;
+		h2 *= m; h2 ^= k2;
+		len -= 4;
+	}
+
+	if(len >= 4)
+	{
+		unsigned int k1 = *data++;
+		k1 *= m; k1 ^= k1 >> r; k1 *= m;
+		h1 *= m; h1 ^= k1;
+		len -= 4;
+	}
+
+	switch(len)
+	{
+	case 3: h2 ^= ((unsigned char*)data)[2] << 16;
+	case 2: h2 ^= ((unsigned char*)data)[1] << 8;
+	case 1: h2 ^= ((unsigned char*)data)[0];
+			h2 *= m;
+	};
+
+	h1 ^= h2 >> 18; h1 *= m;
+	h2 ^= h1 >> 22; h2 *= m;
+	h1 ^= h2 >> 17; h1 *= m;
+	h2 ^= h1 >> 19; h2 *= m;
+
+	uint64_t h = h1;
+
+	h = (h << 32) | h2;
+
+	return h;
+}
+
+VALUE MurmurHashModule = Qnil;
+
+VALUE call_murmur_func
+(unsigned int (*func)(const void*, int, unsigned int), VALUE key, VALUE seed) {
+  int key_length = RSTRING(key)->len;
+  char *key_string = RSTRING(key)->ptr;
+  unsigned int seedling = FIX2UINT(seed);
+
+  unsigned int hash_value = func(key_string, key_length, seedling);
+
+  return UINT2NUM(hash_value);
+}
+
+VALUE call_murmur64_func
+(uint64_t (*func)(const void*, int, unsigned int), VALUE key, VALUE seed) {
+  int key_length = RSTRING(key)->len;
+  char *key_string = RSTRING(key)->ptr;
+  unsigned int seedling = FIX2UINT(seed);
+
+  uint64_t hash_value = func(key_string, key_length, seedling);
+
+  return ULL2NUM(hash_value);
+}
+
+
+VALUE method_murmur_hash(VALUE self, VALUE key, VALUE seed) {
+  return call_murmur_func(MurmurHash2, key, seed);
+}
+
+VALUE method_aligned_murmur_hash(VALUE self, VALUE key, VALUE seed) {
+  return call_murmur_func(MurmurHashAligned2, key, seed);
+}
+
+VALUE method_neutral_murmur_hash(VALUE self, VALUE key, VALUE seed) {
+  return call_murmur_func(MurmurHashNeutral2, key, seed);
+}
+
+VALUE method_murmur_hash64(VALUE self, VALUE key, VALUE seed) {
+#ifdef _LP64
+  /* we're on a 64-bit machine so act like it, punk */
+  return call_murmur64_func(MurmurHash64A, key, seed);
+#else
+  return call_murmur64_func(MurmurHash64B, key, seed);
+#endif
+}
+
+extern "C" void Init_murmur() {
+  MurmurHashModule = rb_define_module("MurmurHash");
+
+  rb_define_module_function(MurmurHashModule,
+                            "murmur_hash",
+                            (VALUE(*)(...))&method_murmur_hash,
+                            2);
+
+  rb_define_module_function(MurmurHashModule,
+                            "aligned_murmur_hash",
+                            (VALUE(*)(...))&method_aligned_murmur_hash,
+                            2);
+  rb_define_module_function(MurmurHashModule,
+                            "neutral_murmur_hash",
+                            (VALUE(*)(...))&method_neutral_murmur_hash,
+                            2);
+  rb_define_module_function(MurmurHashModule,
+                            "murmur_hash64",
+                            (VALUE(*)(...))&method_murmur_hash64,
+                            2);
+}

data/lib/murmur_hash.rb

@@ -0,0 +1,4 @@
+require File.expand_path(File.dirname(__FILE__) + '/murmur')
+module MurmurHash
+  VERSION = '1.0.0'
+end

data/test/test_murmur_hash.rb

@@ -0,0 +1,57 @@
+require "test/unit"
+require "murmur_hash"
+
+
+# I'm sure there's a simpler way to do this, but I can't remember it
+# sooo. From:
+# http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/256730
+# Also posted at: http://gist.github.com/55220
+module Endian
+  def self.big?
+    endianness == :big
+  end
+
+  def self.little?
+    endianness == :little
+  end
+
+  def self.endianness
+    x = 0xdeadbeef
+
+    endian_type = {
+      Array(x).pack("V*") => :little,
+      Array(x).pack("N*") => :big
+    }
+
+    endian_type[Array(x).pack("L*")]
+  end
+end
+
+class TestMurmurHash < Test::Unit::TestCase
+  def test_neutral_does_the_right_thing
+    assert_equal(3435905073, MurmurHash.neutral_murmur_hash("string", 23),
+                 "neutral_murmur_hash returns the same on boxes of both endianesses"
+                 )
+  end
+
+  def test_normal_does_the_right_thing
+    answer = Endian.little? ? 3435905073 : "FIXME find a big-endian box"
+    assert_equal(answer, MurmurHash.murmur_hash("string", 23),
+                 "murmur_hash returns #{answer} on a #{Endian.endianness}-endian box, which this box is"
+                 )
+  end
+
+  def test_aligned_does_the_right_thing
+    answer = Endian.little? ? 3435905073 : "FIXME find a big-endian box"
+    assert_equal(answer, MurmurHash.aligned_murmur_hash("string", 23),
+                 "aligned_murmur_hash returns #{answer} on a #{Endian.endianness}-endian box, which this box is"
+                 )
+  end
+
+  def test_64bit_does_the_right_thing
+    answer = Endian.little? ? 12403831550545192078 : "FIXME find a big-endian box"
+    assert_equal(answer, MurmurHash.murmur_hash64("string", 23),
+                 "64-bit murmur_hash returns #{answer} on a #{Endian.endianness}-endian box, which this box is"
+                 )
+  end
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification 
+name: jmhodges-murmur_hash
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Jeff Hodges
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-05-03 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake-compiler
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "0.5"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.12.2
+    version: 
+description: "* A ruby extension for the MurmurHash2 functions. See http://murmurhash.googlepages.com/"
+email: 
+- jeff@somethingsimilar.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/murmur_hash.rb
+- test/test_murmur_hash.rb
+- ext/murmur/extconf.rb
+- ext/murmur/murmur_hash.cpp
+has_rdoc: true
+homepage: http://github.com/jmhodges/murmur_hash
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: murmur_hash
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 3
+summary: "* A ruby extension for the MurmurHash2 functions"
+test_files: 
+- test/test_murmur_hash.rb