cityhash 0.5.0 → 0.6.0

data/.gitignore

@@ -0,0 +1,7 @@
+*.so
+*.bundle
+.bundle
+Gemfile.lock
+pkg/*
+tmp/*
+vendor/bundle

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3

data/Gemfile

@@ -1,10 +1,7 @@
 source "http://rubygems.org"
 
-gem "ffi"
+gemspec
 
-group :development do
-  gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
-  gem "jeweler", "~> 1.5.2"
-  gem "rcov", ">= 0"
-end
+gem 'rake'
+gem 'rake-compiler'
+gem 'minitest', '~> 2.11.0'

data/README.md

@@ -1,27 +1,29 @@
-### cityhash
+### cityhash [![TravisCI](https://secure.travis-ci.org/nashby/cityhash.png?branch=master)](http://travis-ci.org/nashby/cityhash)
 
 Ruby wrapper for google's cityhash.
 
 ### Install
- 
+
     gem install cityhash
 
 ### Usage
 
-    require 'cityhash'
-    
-    text = "test"
-    seed1 = 12345
-    seed2 = 54321
-    
-    CityHash.hash64(text)               # => 17703940110308125106
-    CityHash.hash64(text, seed1)        # => 14900027982776226655
-    CityHash.hash64(text, seed1, seed2) # => 11136353178704814373
-    CityHash.hash128(text)              # => 130554790001792308794529643941127319555
-    CityHash.hash128(text, seed1)       # => 1238187875708097619810923284135194760226
-	
+```ruby
+require 'cityhash'
+
+text = "test"
+seed1 = 12345
+seed2 = 54321
+
+CityHash.hash64(text)               # => 17703940110308125106
+CityHash.hash64(text, seed1)        # => 14900027982776226655
+CityHash.hash64(text, seed1, seed2) # => 11136353178704814373
+CityHash.hash128(text)              # => 1800071687761605184910580728449884026697
+CityHash.hash128(text, seed1)       # => 6087407617808651818174120599816915369
+```
+
 ### Contributing to cityhash
- 
+
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
 * Fork the project
@@ -34,10 +36,13 @@ Ruby wrapper for google's cityhash.
 
 #### [Contributors](http://github.com/nashby/cityhash/contributors)
  - [Johannes Holzfuß](http://github.com/DataWraith)
+ - [Quin Hoxie](https://github.com/qhoxie)
+ - [David Dai](https://github.com/newtonapple)
+ - [Sergey Nartimov](https://github.com/lest)
 
 [Vasiliy Ermolovich](http://github.com/nashby/)<br/>
 
 ### Copyright
 
-Copyright (c) 2011 nashby. See LICENSE.txt for
+Copyright (c) 2012 Vasiliy Ermolovich. See LICENSE.txt for
 further details.

data/Rakefile

@@ -1,54 +1,20 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-
-$LOAD_PATH.unshift('lib')
-
-require 'jeweler'
-require 'cityhash/version'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
-  gem.name = "cityhash"
-  gem.homepage = "http://github.com/nashby/cityhash"
-  gem.license = "MIT"
-  gem.version = CityHash::Version::STRING
-  gem.summary = "ffi wrapper for google's cityhash"
-  gem.description = "ffi wrapper for google's cityhash"
-  gem.email = "younash@gmail.com"
-  gem.authors = ["nashby"]
-  gem.add_runtime_dependency 'ffi'
-end
-Jeweler::RubygemsDotOrgTasks.new
+require "bundler/gem_tasks"
 
 require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
+require 'rake/clean'
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = 'test/*_test.rb'
+  t.verbose = true
 end
 
+desc "Run tests"
 task :default => :test
 
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "cityhash #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
+require 'rake/extensiontask'
+Rake::ExtensionTask.new('cityhash') do |ext|
+  ext.lib_dir = 'lib/cityhash'
 end
+
+Rake::Task[:test].prerequisites << :compile

data/cityhash.gemspec

@@ -1,75 +1,21 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "cityhash/version"
 
 Gem::Specification.new do |s|
-  s.name = %q{cityhash}
-  s.version = "0.5.0"
+  s.name        = "cityhash"
+  s.version     = CityHash::VERSION
+  s.authors     = ["Vasiliy Ermolovich"]
+  s.email       = ["younash@gmail.com"]
+  s.homepage    = "http://github.com/nashby/cityhash"
+  s.summary     = %q{ruby bindings for google's cityhash}
+  s.description = %q{ruby bindings for google's cityhash}
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = [%q{nashby}]
-  s.date = %q{2011-10-28}
-  s.description = %q{ffi wrapper for google's cityhash}
-  s.email = %q{younash@gmail.com}
-  s.extensions = [%q{ext/cityhash/extconf.rb}]
-  s.extra_rdoc_files = [
-    "LICENSE.txt",
-    "README.md"
-  ]
-  s.files = [
-    ".document",
-    "Gemfile",
-    "Gemfile.lock",
-    "LICENSE.txt",
-    "README.md",
-    "Rakefile",
-    "cityhash.gemspec",
-    "ext/cityhash/city.cc",
-    "ext/cityhash/city.h",
-    "ext/cityhash/city_ruby_bridge.cpp",
-    "ext/cityhash/city_ruby_bridge.h",
-    "ext/cityhash/extconf.rb",
-    "lib/cityhash.rb",
-    "lib/cityhash/version.rb",
-    "test/helper.rb",
-    "test/test_cityhash.rb"
-  ]
-  s.homepage = %q{http://github.com/nashby/cityhash}
-  s.licenses = [%q{MIT}]
-  s.require_paths = [%q{lib}]
-  s.rubygems_version = %q{1.8.6}
-  s.summary = %q{ffi wrapper for google's cityhash}
-  s.test_files = [
-    "test/helper.rb",
-    "test/test_cityhash.rb"
-  ]
+  s.rubyforge_project = "cityhash"
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<ffi>, [">= 0"])
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
-      s.add_runtime_dependency(%q<ffi>, [">= 0"])
-    else
-      s.add_dependency(%q<ffi>, [">= 0"])
-      s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
-      s.add_dependency(%q<rcov>, [">= 0"])
-      s.add_dependency(%q<ffi>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<ffi>, [">= 0"])
-    s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
-    s.add_dependency(%q<rcov>, [">= 0"])
-    s.add_dependency(%q<ffi>, [">= 0"])
-  end
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.extensions    = ["ext/cityhash/extconf.rb"]
 end
-

data/ext/cityhash/city.cc

@@ -18,7 +18,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 //
-// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala
+// CityHash, by Geoff Pike and Jyrki Alakuijala
 //
 // This file provides CityHash64() and related functions.
 //
@@ -30,20 +30,65 @@
 #include "city.h"
 
 #include <algorithm>
+#include <string.h>  // for memcpy and memset
 
 using namespace std;
 
-#define UNALIGNED_LOAD64(p) (*(const uint64*)(p))
-#define UNALIGNED_LOAD32(p) (*(const uint32*)(p))
+static uint64 UNALIGNED_LOAD64(const char *p) {
+  uint64 result;
+  memcpy(&result, p, sizeof(result));
+  return result;
+}
+
+static uint32 UNALIGNED_LOAD32(const char *p) {
+  uint32 result;
+  memcpy(&result, p, sizeof(result));
+  return result;
+}
+
+#if !defined(WORDS_BIGENDIAN)
+
+#define uint32_in_expected_order(x) (x)
+#define uint64_in_expected_order(x) (x)
+
+#else
+
+#ifdef _MSC_VER
+#include <stdlib.h>
+#define bswap_32(x) _byteswap_ulong(x)
+#define bswap_64(x) _byteswap_uint64(x)
+
+#elif defined(__APPLE__)
+// Mac OS X / Darwin features
+#include <libkern/OSByteOrder.h>
+#define bswap_32(x) OSSwapInt32(x)
+#define bswap_64(x) OSSwapInt64(x)
+
+#else
+#include <byteswap.h>
+#endif
+
+#define uint32_in_expected_order(x) (bswap_32(x))
+#define uint64_in_expected_order(x) (bswap_64(x))
+
+#endif  // WORDS_BIGENDIAN
 
 #if !defined(LIKELY)
-#if defined(__GNUC__)
+#if HAVE_BUILTIN_EXPECT
 #define LIKELY(x) (__builtin_expect(!!(x), 1))
 #else
 #define LIKELY(x) (x)
 #endif
 #endif
 
+static uint64 Fetch64(const char *p) {
+  return uint64_in_expected_order(UNALIGNED_LOAD64(p));
+}
+
+static uint32 Fetch32(const char *p) {
+  return uint32_in_expected_order(UNALIGNED_LOAD32(p));
+}
+
 // Some primes between 2^63 and 2^64 for various uses.
 static const uint64 k0 = 0xc3a5c85c97cb3127ULL;
 static const uint64 k1 = 0xb492b66fbe98f273ULL;
@@ -74,13 +119,13 @@ static uint64 HashLen16(uint64 u, uint64 v) {
 
 static uint64 HashLen0to16(const char *s, size_t len) {
   if (len > 8) {
-    uint64 a = UNALIGNED_LOAD64(s);
-    uint64 b = UNALIGNED_LOAD64(s + len - 8);
+    uint64 a = Fetch64(s);
+    uint64 b = Fetch64(s + len - 8);
     return HashLen16(a, RotateByAtLeast1(b + len, len)) ^ b;
   }
   if (len >= 4) {
-    uint64 a = UNALIGNED_LOAD32(s);
-    return HashLen16(len + (a << 3), UNALIGNED_LOAD32(s + len - 4));
+    uint64 a = Fetch32(s);
+    return HashLen16(len + (a << 3), Fetch32(s + len - 4));
   }
   if (len > 0) {
     uint8 a = s[0];
@@ -96,10 +141,10 @@ static uint64 HashLen0to16(const char *s, size_t len) {
 // This probably works well for 16-byte strings as well, but it may be overkill
 // in that case.
 static uint64 HashLen17to32(const char *s, size_t len) {
-  uint64 a = UNALIGNED_LOAD64(s) * k1;
-  uint64 b = UNALIGNED_LOAD64(s + 8);
-  uint64 c = UNALIGNED_LOAD64(s + len - 8) * k2;
-  uint64 d = UNALIGNED_LOAD64(s + len - 16) * k0;
+  uint64 a = Fetch64(s) * k1;
+  uint64 b = Fetch64(s + 8);
+  uint64 c = Fetch64(s + len - 8) * k2;
+  uint64 d = Fetch64(s + len - 16) * k0;
   return HashLen16(Rotate(a - b, 43) + Rotate(c, 30) + d,
                    a + Rotate(b ^ k3, 20) - c + len);
 }
@@ -120,32 +165,32 @@ static pair<uint64, uint64> WeakHashLen32WithSeeds(
 // Return a 16-byte hash for s[0] ... s[31], a, and b.  Quick and dirty.
 static pair<uint64, uint64> WeakHashLen32WithSeeds(
     const char* s, uint64 a, uint64 b) {
-  return WeakHashLen32WithSeeds(UNALIGNED_LOAD64(s),
-                                UNALIGNED_LOAD64(s + 8),
-                                UNALIGNED_LOAD64(s + 16),
-                                UNALIGNED_LOAD64(s + 24),
+  return WeakHashLen32WithSeeds(Fetch64(s),
+                                Fetch64(s + 8),
+                                Fetch64(s + 16),
+                                Fetch64(s + 24),
                                 a,
                                 b);
 }
 
 // Return an 8-byte hash for 33 to 64 bytes.
 static uint64 HashLen33to64(const char *s, size_t len) {
-  uint64 z = UNALIGNED_LOAD64(s + 24);
-  uint64 a = UNALIGNED_LOAD64(s) + (len + UNALIGNED_LOAD64(s + len - 16)) * k0;
+  uint64 z = Fetch64(s + 24);
+  uint64 a = Fetch64(s) + (len + Fetch64(s + len - 16)) * k0;
   uint64 b = Rotate(a + z, 52);
   uint64 c = Rotate(a, 37);
-  a += UNALIGNED_LOAD64(s + 8);
+  a += Fetch64(s + 8);
   c += Rotate(a, 7);
-  a += UNALIGNED_LOAD64(s + 16);
+  a += Fetch64(s + 16);
   uint64 vf = a + z;
   uint64 vs = b + Rotate(a, 31) + c;
-  a = UNALIGNED_LOAD64(s + 16) + UNALIGNED_LOAD64(s + len - 32);
-  z = UNALIGNED_LOAD64(s + len - 8);
+  a = Fetch64(s + 16) + Fetch64(s + len - 32);
+  z = Fetch64(s + len - 8);
   b = Rotate(a + z, 52);
   c = Rotate(a, 37);
-  a += UNALIGNED_LOAD64(s + len - 24);
+  a += Fetch64(s + len - 24);
   c += Rotate(a, 7);
-  a += UNALIGNED_LOAD64(s + len - 16);
+  a += Fetch64(s + len - 16);
   uint64 wf = a + z;
   uint64 ws = b + Rotate(a, 31) + c;
   uint64 r = ShiftMix((vf + ws) * k2 + (wf + vs) * k0);
@@ -165,25 +210,23 @@ uint64 CityHash64(const char *s, size_t len) {
 
   // For strings over 64 bytes we hash the end first, and then as we
   // loop we keep 56 bytes of state: v, w, x, y, and z.
-  uint64 x = UNALIGNED_LOAD64(s);
-  uint64 y = UNALIGNED_LOAD64(s + len - 16) ^ k1;
-  uint64 z = UNALIGNED_LOAD64(s + len - 56) ^ k0;
-  pair<uint64, uint64> v = WeakHashLen32WithSeeds(s + len - 64, len, y);
-  pair<uint64, uint64> w = WeakHashLen32WithSeeds(s + len - 32, len * k1, k0);
-  z += ShiftMix(v.second) * k1;
-  x = Rotate(z + x, 39) * k1;
-  y = Rotate(y, 33) * k1;
+  uint64 x = Fetch64(s + len - 40);
+  uint64 y = Fetch64(s + len - 16) + Fetch64(s + len - 56);
+  uint64 z = HashLen16(Fetch64(s + len - 48) + len, Fetch64(s + len - 24));
+  pair<uint64, uint64> v = WeakHashLen32WithSeeds(s + len - 64, len, z);
+  pair<uint64, uint64> w = WeakHashLen32WithSeeds(s + len - 32, y + k1, x);
+  x = x * k1 + Fetch64(s);
 
   // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
   len = (len - 1) & ~static_cast<size_t>(63);
   do {
-    x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
-    y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
+    x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
+    y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
     x ^= w.second;
-    y ^= v.first;
-    z = Rotate(z ^ w.first, 33);
+    y += v.first + Fetch64(s + 40);
+    z = Rotate(z + w.first, 33) * k1;
     v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
-    w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
+    w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
     std::swap(z, x);
     s += 64;
     len -= 64;
@@ -202,25 +245,26 @@ uint64 CityHash64WithSeeds(const char *s, size_t len,
 }
 
 // A subroutine for CityHash128().  Returns a decent 128-bit hash for strings
-// of any length representable in ssize_t.  Based on City and Murmur.
+// of any length representable in signed long.  Based on City and Murmur.
 static uint128 CityMurmur(const char *s, size_t len, uint128 seed) {
   uint64 a = Uint128Low64(seed);
   uint64 b = Uint128High64(seed);
   uint64 c = 0;
   uint64 d = 0;
-  ssize_t l = len - 16;
+  signed long l = len - 16;
   if (l <= 0) {  // len <= 16
+    a = ShiftMix(a * k1) * k1;
     c = b * k1 + HashLen0to16(s, len);
-    d = Rotate(a + (len >= 8 ? UNALIGNED_LOAD64(s) : c), 32);
+    d = ShiftMix(a + (len >= 8 ? Fetch64(s) : c));
   } else {  // len > 16
-    c = HashLen16(UNALIGNED_LOAD64(s + len - 8) + k1, a);
-    d = HashLen16(b + len, c + UNALIGNED_LOAD64(s + len - 16));
+    c = HashLen16(Fetch64(s + len - 8) + k1, a);
+    d = HashLen16(b + len, c + Fetch64(s + len - 16));
     a += d;
     do {
-      a ^= ShiftMix(UNALIGNED_LOAD64(s) * k1) * k1;
+      a ^= ShiftMix(Fetch64(s) * k1) * k1;
       a *= k1;
       b ^= a;
-      c ^= ShiftMix(UNALIGNED_LOAD64(s + 8) * k1) * k1;
+      c ^= ShiftMix(Fetch64(s + 8) * k1) * k1;
       c *= k1;
       d ^= c;
       s += 16;
@@ -243,49 +287,50 @@ uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed) {
   uint64 x = Uint128Low64(seed);
   uint64 y = Uint128High64(seed);
   uint64 z = len * k1;
-  v.first = Rotate(y ^ k1, 49) * k1 + UNALIGNED_LOAD64(s);
-  v.second = Rotate(v.first, 42) * k1 + UNALIGNED_LOAD64(s + 8);
+  v.first = Rotate(y ^ k1, 49) * k1 + Fetch64(s);
+  v.second = Rotate(v.first, 42) * k1 + Fetch64(s + 8);
   w.first = Rotate(y + z, 35) * k1 + x;
-  w.second = Rotate(x + UNALIGNED_LOAD64(s + 88), 53) * k1;
+  w.second = Rotate(x + Fetch64(s + 88), 53) * k1;
 
   // This is the same inner loop as CityHash64(), manually unrolled.
   do {
-    x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
-    y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
+    x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
+    y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
     x ^= w.second;
-    y ^= v.first;
-    z = Rotate(z ^ w.first, 33);
+    y += v.first + Fetch64(s + 40);
+    z = Rotate(z + w.first, 33) * k1;
     v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
-    w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
+    w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
     std::swap(z, x);
     s += 64;
-    x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1;
-    y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1;
+    x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
+    y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
     x ^= w.second;
-    y ^= v.first;
-    z = Rotate(z ^ w.first, 33);
+    y += v.first + Fetch64(s + 40);
+    z = Rotate(z + w.first, 33) * k1;
     v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
-    w = WeakHashLen32WithSeeds(s + 32, z + w.second, y);
+    w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
     std::swap(z, x);
     s += 64;
     len -= 128;
   } while (LIKELY(len >= 128));
-  y += Rotate(w.first, 37) * k0 + z;
   x += Rotate(v.first + z, 49) * k0;
+  z += Rotate(w.first, 37) * k0;
   // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s.
   for (size_t tail_done = 0; tail_done < len; ) {
     tail_done += 32;
-    y = Rotate(y - x, 42) * k0 + v.second;
-    w.first += UNALIGNED_LOAD64(s + len - tail_done + 16);
-    x = Rotate(x, 49) * k0 + w.first;
-    w.first += v.first;
-    v = WeakHashLen32WithSeeds(s + len - tail_done, v.first, v.second);
+    y = Rotate(x + y, 42) * k0 + v.second;
+    w.first += Fetch64(s + len - tail_done + 16);
+    x = x * k0 + w.first;
+    z += w.second + Fetch64(s + len - tail_done);
+    w.second += v.first;
+    v = WeakHashLen32WithSeeds(s + len - tail_done, v.first + z, v.second);
   }
-  // At this point our 48 bytes of state should contain more than
+  // At this point our 56 bytes of state should contain more than
   // enough information for a strong 128-bit hash.  We use two
-  // different 48-byte-to-8-byte hashes to get a 16-byte final result.
+  // different 56-byte-to-8-byte hashes to get a 16-byte final result.
   x = HashLen16(x, v.first);
-  y = HashLen16(y, w.first);
+  y = HashLen16(y + z, w.first);
   return uint128(HashLen16(x + v.second, w.second) + y,
                  HashLen16(x + w.second, y + v.second));
 }
@@ -294,14 +339,128 @@ uint128 CityHash128(const char *s, size_t len) {
   if (len >= 16) {
     return CityHash128WithSeed(s + 16,
                                len - 16,
-                               uint128(UNALIGNED_LOAD64(s) ^ k3,
-                                       UNALIGNED_LOAD64(s + 8)));
+                               uint128(Fetch64(s) ^ k3,
+                                       Fetch64(s + 8)));
   } else if (len >= 8) {
     return CityHash128WithSeed(NULL,
                                0,
-                               uint128(UNALIGNED_LOAD64(s) ^ (len * k0),
-                                       UNALIGNED_LOAD64(s + len - 8) ^ k1));
+                               uint128(Fetch64(s) ^ (len * k0),
+                                       Fetch64(s + len - 8) ^ k1));
   } else {
     return CityHash128WithSeed(s, len, uint128(k0, k1));
   }
 }
+
+#ifdef __SSE4_2__
+#include <citycrc.h>
+#include <nmmintrin.h>
+
+// Requires len >= 240.
+static void CityHashCrc256Long(const char *s, size_t len,
+                               uint32 seed, uint64 *result) {
+  uint64 a = Fetch64(s + 56) + k0;
+  uint64 b = Fetch64(s + 96) + k0;
+  uint64 c = result[0] = HashLen16(b, len);
+  uint64 d = result[1] = Fetch64(s + 120) * k0 + len;
+  uint64 e = Fetch64(s + 184) + seed;
+  uint64 f = seed;
+  uint64 g = 0;
+  uint64 h = 0;
+  uint64 i = 0;
+  uint64 j = 0;
+  uint64 t = c + d;
+
+  // 240 bytes of input per iter.
+  size_t iters = len / 240;
+  len -= iters * 240;
+  do {
+#define CHUNK(multiplier, z)                                    \
+    {                                                           \
+      uint64 old_a = a;                                         \
+      a = Rotate(b, 41 ^ z) * multiplier + Fetch64(s);          \
+      b = Rotate(c, 27 ^ z) * multiplier + Fetch64(s + 8);      \
+      c = Rotate(d, 41 ^ z) * multiplier + Fetch64(s + 16);     \
+      d = Rotate(e, 33 ^ z) * multiplier + Fetch64(s + 24);     \
+      e = Rotate(t, 25 ^ z) * multiplier + Fetch64(s + 32);     \
+      t = old_a;                                                \
+    }                                                           \
+    f = _mm_crc32_u64(f, a);                                    \
+    g = _mm_crc32_u64(g, b);                                    \
+    h = _mm_crc32_u64(h, c);                                    \
+    i = _mm_crc32_u64(i, d);                                    \
+    j = _mm_crc32_u64(j, e);                                    \
+    s += 40
+
+    CHUNK(1, 1); CHUNK(k0, 0);
+    CHUNK(1, 1); CHUNK(k0, 0);
+    CHUNK(1, 1); CHUNK(k0, 0);
+  } while (--iters > 0);
+
+  while (len >= 40) {
+    CHUNK(k0, 0);
+    len -= 40;
+  }
+  if (len > 0) {
+    s = s + len - 40;
+    CHUNK(k0, 0);
+  }
+  j += i << 32;
+  a = HashLen16(a, j);
+  h += g << 32;
+  b += h;
+  c = HashLen16(c, f) + i;
+  d = HashLen16(d, e + result[0]);
+  j += e;
+  i += HashLen16(h, t);
+  e = HashLen16(a, d) + j;
+  f = HashLen16(b, c) + a;
+  g = HashLen16(j, i) + c;
+  result[0] = e + f + g + h;
+  a = ShiftMix((a + g) * k0) * k0 + b;
+  result[1] += a + result[0];
+  a = ShiftMix(a * k0) * k0 + c;
+  result[2] = a + result[1];
+  a = ShiftMix((a + e) * k0) * k0;
+  result[3] = a + result[2];
+}
+
+// Requires len < 240.
+static void CityHashCrc256Short(const char *s, size_t len, uint64 *result) {
+  char buf[240];
+  memcpy(buf, s, len);
+  memset(buf + len, 0, 240 - len);
+  CityHashCrc256Long(buf, 240, ~static_cast<uint32>(len), result);
+}
+
+void CityHashCrc256(const char *s, size_t len, uint64 *result) {
+  if (LIKELY(len >= 240)) {
+    CityHashCrc256Long(s, len, 0, result);
+  } else {
+    CityHashCrc256Short(s, len, result);
+  }
+}
+
+uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed) {
+  if (len <= 900) {
+    return CityHash128WithSeed(s, len, seed);
+  } else {
+    uint64 result[4];
+    CityHashCrc256(s, len, result);
+    uint64 u = Uint128High64(seed) + result[0];
+    uint64 v = Uint128Low64(seed) + result[1];
+    return uint128(HashLen16(u, v + result[2]),
+                   HashLen16(Rotate(v, 32), u * k0 + result[3]));
+  }
+}
+
+uint128 CityHashCrc128(const char *s, size_t len) {
+  if (len <= 900) {
+    return CityHash128(s, len);
+  } else {
+    uint64 result[4];
+    CityHashCrc256(s, len, result);
+    return uint128(result[2], result[3]);
+  }
+}
+
+#endif

data/ext/cityhash/city.h

@@ -18,7 +18,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 //
-// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala
+// CityHash, by Geoff Pike and Jyrki Alakuijala
 //
 // This file provides a few functions for hashing strings. On x86-64
 // hardware in 2011, CityHash64() is faster than other high-quality

data/ext/cityhash/cityhash.cc

@@ -0,0 +1,47 @@
+#include <ruby.h>
+#include "city.h"
+
+// Use this typedef to make the compiler happy when
+// calling rb_define_method()
+typedef VALUE (ruby_method)(...);
+
+extern "C" VALUE cityhash_hash64(VALUE mod, VALUE input)
+{
+  return ULL2NUM(CityHash64(StringValuePtr(input), RSTRING_LEN(input)));
+}
+
+extern "C" VALUE cityhash_hash64_with_seed(VALUE mod, VALUE input, VALUE seed)
+{
+  return ULL2NUM(CityHash64WithSeed(StringValuePtr(input), RSTRING_LEN(input), NUM2ULL(seed)));
+}
+
+extern "C" VALUE cityhash_hash64_with_seeds(VALUE mod, VALUE input, VALUE seed1, VALUE seed2)
+{
+  return ULL2NUM(CityHash64WithSeeds(StringValuePtr(input), RSTRING_LEN(input), NUM2ULL(seed1), NUM2ULL(seed2)));
+}
+
+extern "C" VALUE cityhash_hash128(VALUE mod, VALUE input)
+{
+  uint128 hash = CityHash128(StringValuePtr(input), RSTRING_LEN(input));
+  return rb_str_new((char *)&hash, sizeof(hash));
+}
+
+extern "C" VALUE cityhash_hash128_with_seed(VALUE mod, VALUE input, VALUE seed_string)
+{
+  uint128 seed = *(uint128 *)StringValuePtr(seed_string);
+  uint128 hash = CityHash128WithSeed(StringValuePtr(input), RSTRING_LEN(input), seed);
+  return rb_str_new((char *)&hash, sizeof(hash));
+}
+
+extern "C" void Init_cityhash()
+{
+  VALUE mCityHash = rb_define_module("CityHash");
+  VALUE mInternal = rb_define_module_under(mCityHash, "Internal");
+
+  rb_define_singleton_method(mInternal, "hash64", (ruby_method*) &cityhash_hash64, 1);
+  rb_define_singleton_method(mInternal, "hash64_with_seed", (ruby_method*) &cityhash_hash64_with_seed, 2);
+  rb_define_singleton_method(mInternal, "hash64_with_seeds",  (ruby_method*) &cityhash_hash64_with_seeds, 3);
+
+  rb_define_singleton_method(mInternal, "hash128", (ruby_method*) &cityhash_hash128, 1);
+  rb_define_singleton_method(mInternal, "hash128_with_seed", (ruby_method*) &cityhash_hash128_with_seed, 2);
+}

data/ext/cityhash/extconf.rb

@@ -1,12 +1,8 @@
 require 'mkmf'
 
-have_library('objc') if RUBY_PLATFORM =~ /darwin/
-
-%w{g O3 Wall rdynamic}.each do |flag|
+%w{g O3 Wall}.each do |flag|
   flag = "-#{flag}"
   $CPPFLAGS += " #{flag}" unless $CPPFLAGS.split.include? flag
 end
 
-CONFIG['LDSHARED'] = '$(CXX) -shared' unless RUBY_PLATFORM =~ /darwin/
-
-create_makefile('cityhash')
+create_makefile('cityhash/cityhash')

data/lib/cityhash.rb

@@ -1,63 +1,23 @@
-require 'ffi'
+require 'cityhash/version'
+require 'cityhash/cityhash'
 
 module CityHash
-
   LOW64_MASK  = 0x0000000000000000ffffffffffffffff
   HIGH64_MASK = 0xffffffffffffffff0000000000000000
 
-  module Internal
-    extend FFI::Library
-
-    ffi_lib(
-      case RUBY_PLATFORM
-        when /darwin/
-          File.join(File.dirname(__FILE__), '..', 'ext', 'cityhash', 'cityhash.bundle')
-        when /mingw|mswin|linux/
-          File.join(File.dirname(__FILE__), '..', 'ext', 'cityhash', 'cityhash.so')
-        else
-          File.join(File.dirname(__FILE__), '..', 'ext', 'cityhash', "cityhash.#{RbConfig::CONFIG['DLEXT']}")
-      end
-    )
-
-    attach_function :city_hash64, :CityHash64, [:string, :size_t], :uint64
-    attach_function :city_hash64_with_seed, :CityHash64WithSeed, [:string, :size_t, :uint64], :uint64
-    attach_function :city_hash64_with_seeds, :CityHash64WithSeeds, [:string, :size_t, :uint64, :uint64], :uint64
-    attach_function :city_hash128, :CityHash128String, [:string, :size_t], :string
-    attach_function :city_hash128_with_seed, :CityHash128WithSeedString, [:string, :size_t, :uint64, :uint64], :string
+  def self.hash64(input, seed1=nil, seed2=nil)
+    return Internal.hash64(input) if seed1.nil?
+    return Internal.hash64_with_seed(input, seed1.to_i) if seed2.nil?
+    Internal.hash64_with_seeds(input, seed1.to_i, seed2.to_i)
   end
 
-  def self.hash64(input, seed1 = nil, seed2 = nil)
-    input_str = input.to_s
-
-    # Ruby 1.8 compatibility
-    if input_str.respond_to?(:bytesize)
-      len = input_str.bytesize
+  def self.hash128(input, seed=nil)
+    if seed
+      seed = [seed & LOW64_MASK, seed & HIGH64_MASK >> 64].pack('QQ')
+      digest = Internal.hash128_with_seed(input, seed)
     else
-      len = input_str.size
+      digest = Internal.hash128(input)
     end
-
-    return CityHash::Internal.city_hash64(input_str, len) if seed1.nil? 
-    return CityHash::Internal.city_hash64_with_seed(input_str, len, seed1.to_i & LOW64_MASK) if seed2.nil?
-
-    CityHash::Internal.city_hash64_with_seeds(input_str, len, seed1.to_i & LOW64_MASK, seed2.to_i & LOW64_MASK)
+    [0..7, 8..15].map { |r| digest[r].unpack('Q').first.to_s }.join.to_i
   end
-
-  def self.hash128(input, seed = nil)
-    input_str = input.to_s
-
-    # Ruby 1.8 compatibility
-    if input_str.respond_to?(:bytesize)
-      len = input_str.bytesize
-    else
-      len = input_str.size
-    end
-
-    return CityHash::Internal.city_hash128(input_str, len).to_i if seed.nil?
-
-    seed_low  =  seed.to_i & LOW64_MASK
-    seed_high = (seed.to_i & HIGH64_MASK) >> 64
-
-    CityHash::Internal.city_hash128_with_seed(input_str, len, seed_low, seed_high).to_i
-  end
-
 end

data/lib/cityhash/version.rb

@@ -1,12 +1,3 @@
 module CityHash
-
-  module Version
-    MAJOR = 0
-    MINOR = 5
-    PATCH = 0
-    BUILD = nil
-
-    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
-  end
-
+  VERSION = "0.6.0"
 end

data/test/cityhash_test.rb

@@ -0,0 +1,24 @@
+require 'test_helper'
+
+describe CityHash do
+  it 'returns 64bit hash' do
+    assert_equal 17703940110308125106, CityHash.hash64("test")
+  end
+
+  it "returns 64bit hash with a seed" do
+    assert_equal 14900027982776226655, CityHash.hash64("test", 12345)
+  end
+
+  it "returns 64bit hash with seeds" do
+    assert_equal 11136353178704814373, CityHash.hash64("test", 12345, 54321)
+  end
+
+  it "returns 128bit hash" do
+    assert_equal 1800071687761605184910580728449884026697, CityHash.hash128("test")
+  end
+
+  it "returns 128bit hash with seed" do
+    seed = (123 << 64) | 123
+    assert_equal 1631427474705635869517741677842296176559, CityHash.hash128("test", seed)
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require 'minitest/autorun'
+require 'minitest/spec'
+
+require 'cityhash'

metadata

@@ -1,110 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: cityhash
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
-- nashby
+- Vasiliy Ermolovich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-28 00:00:00.000000000Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: ffi
-  requirement: &24780520 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *24780520
-- !ruby/object:Gem::Dependency
-  name: shoulda
-  requirement: &24779860 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *24779860
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: &24779220 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :development
-  prerelease: false
-  version_requirements: *24779220
-- !ruby/object:Gem::Dependency
-  name: jeweler
-  requirement: &24778500 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.5.2
-  type: :development
-  prerelease: false
-  version_requirements: *24778500
-- !ruby/object:Gem::Dependency
-  name: rcov
-  requirement: &24777780 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *24777780
-- !ruby/object:Gem::Dependency
-  name: ffi
-  requirement: &24776560 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *24776560
-description: ffi wrapper for google's cityhash
-email: younash@gmail.com
+date: 2012-04-07 00:00:00.000000000Z
+dependencies: []
+description: ruby bindings for google's cityhash
+email:
+- younash@gmail.com
 executables: []
 extensions:
 - ext/cityhash/extconf.rb
-extra_rdoc_files:
-- LICENSE.txt
-- README.md
+extra_rdoc_files: []
 files:
-- .document
+- .gitignore
+- .travis.yml
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - cityhash.gemspec
 - ext/cityhash/city.cc
 - ext/cityhash/city.h
-- ext/cityhash/city_ruby_bridge.cpp
-- ext/cityhash/city_ruby_bridge.h
+- ext/cityhash/cityhash.cc
 - ext/cityhash/extconf.rb
 - lib/cityhash.rb
 - lib/cityhash/version.rb
-- test/helper.rb
-- test/test_cityhash.rb
+- test/cityhash_test.rb
+- test/test_helper.rb
 homepage: http://github.com/nashby/cityhash
-licenses:
-- MIT
+licenses: []
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -117,19 +48,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4372501756799673446
+      hash: 268501841747995875
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 268501841747995875
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.6
+rubyforge_project: cityhash
+rubygems_version: 1.8.19
 signing_key: 
 specification_version: 3
-summary: ffi wrapper for google's cityhash
-test_files:
-- test/helper.rb
-- test/test_cityhash.rb
+summary: ruby bindings for google's cityhash
+test_files: []

data/.document

@@ -1,5 +0,0 @@
-lib/**/*.rb
-bin/*
-- 
-features/**/*.feature
-LICENSE.txt

data/Gemfile.lock

@@ -1,23 +0,0 @@
-GEM
-  remote: http://rubygems.org/
-  specs:
-    ffi (1.0.7)
-      rake (>= 0.8.7)
-    git (1.2.5)
-    jeweler (1.5.2)
-      bundler (~> 1.0.0)
-      git (>= 1.2.5)
-      rake
-    rake (0.8.7)
-    rcov (0.9.9)
-    shoulda (2.11.3)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.0.0)
-  ffi
-  jeweler (~> 1.5.2)
-  rcov
-  shoulda

data/ext/cityhash/city_ruby_bridge.cpp

@@ -1,19 +0,0 @@
-#include "city_ruby_bridge.h"
-#include <sstream>
-
-const char* CityHash128String(const char *s, size_t len)
-{
-  std::ostringstream str;
-  str << Uint128Low64(CityHash128(s, len)) << Uint128High64(CityHash128(s, len));
-
-  return str.str().c_str();
-}
-
-const char* CityHash128WithSeedString(const char *s, size_t len, uint64 seed1, uint64 seed2)
-{
-  uint128 seed(seed1, seed2);
-  std::ostringstream str;
-  str << Uint128Low64(CityHash128WithSeed(s, len, seed)) << Uint128High64(CityHash128WithSeed(s, len, seed));
-
-  return str.str().c_str();
-}

data/ext/cityhash/city_ruby_bridge.h

@@ -1,8 +0,0 @@
-#include "city.h"
-
-typedef uint64_t uint64;
-typedef std::pair<uint64, uint64> uint128;
-
-extern "C" const char* CityHash128String(const char *s, size_t len);
-
-extern "C" const char* CityHash128WithSeedString(const char *s, size_t len, uint64 seed1, uint64 seed2);

data/test/helper.rb

@@ -1,18 +0,0 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-require 'shoulda'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'cityhash'
-
-class Test::Unit::TestCase
-end

data/test/test_cityhash.rb

@@ -1,26 +0,0 @@
-require 'helper'
-
-class TestCityhash < Test::Unit::TestCase
-
-  should "return 64bit hash" do
-    assert_equal 17703940110308125106, CityHash.hash64("test")
-  end
-
-  should "return 64bit hash with a seed" do
-    assert_equal 14900027982776226655, CityHash.hash64("test", 12345)
-  end
-
-  should "return 64bit hash with seeds" do
-    assert_equal 11136353178704814373, CityHash.hash64("test", 12345, 54321)
-  end
-
-  should "return 128bit hash" do
-    assert_equal 130554790001792308794529643941127319555, CityHash.hash128("test")
-  end
-
-  should "return 128bit hash with seed" do
-    seed = (123 << 64) | 123
-    assert_equal 183946415266487905135364192266033899899, CityHash.hash128("test", seed)
-  end
-
-end