hyperll 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 08b6c537f72013125c31cab21a5fc92592d5e3fc
+  data.tar.gz: 4ceb4deff0578637df2c0330347fc530b676aa81
+SHA512:
+  metadata.gz: 61bb663e66ccf133d5db20518961867efb350f2a372227de0f49c1c28ac32484c7fd609024274d6fbea9995a39a8ce9e47e8977ae5b91c2c05242dbc1365849d
+  data.tar.gz: 3ce7d45798faf5c98a779767760fd325edbe015fe7e8537a063aa25b00d85e123006f3d1be6158ebd023904e0365b18a4a4b195cae32b2cbc22d3a5026700f69

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hyperll.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Andy Lindeman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# Hyperll
+
+HyperLogLog implementation in pure Ruby
+
+## Usage
+
+HyperLogLog stores an estimation of the cardinality of a set. It can be merged
+with other HyperLogLog instances.
+
+```ruby
+hll = Hyperll::HyperLogLog.new(10)
+hll.offer(1)
+hll.offer(2)
+hll.offer(3)
+hll.cardinality # => 3
+
+hll2 = Hyperll::HyperLogLog.new(10)
+hll2.offer(3)
+hll2.offer(4)
+hll2.offer(5)
+hll.cardinality # => 3
+
+merged = Hyperll::HyperLogLog.new(10)
+merged.merge(hll, hll2)
+merged.cardinality # => 5
+```
+
+### Serialization
+
+HyperLogLog can be serialized to a binary string. It is compatible with the
+binary format from the Java [stream-lib](https://github.com/addthis/stream-lib)
+library.
+
+```ruby
+hll = Hyperll::HyperLogLog.new(4)
+hll.offer(1)
+hll.offer(2)
+hll.offer(3)
+hll.serialize # => "\x00\x00\x00\x04\x00\x00\x00\f\x02\x00\x00\x00\x00\x00\x88\x00\x00\x00\x00\x00"
+
+hll2 = Hyperll::HyperLogLog.unserialize("\x00\x00\x00\x04\x00\x00\x00\f\x02\x00\x00\x00\x00\x00\x88\x00\x00\x00\x00\x00")
+hll2.cardinality # => 3
+```

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/hyperll.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hyperll/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hyperll"
+  spec.version       = Hyperll::VERSION
+  spec.authors       = ["Andy Lindeman"]
+  spec.email         = ["andy@andylindeman.com"]
+  spec.description   = %q{HyperLogLog implementation in pure Ruby}
+  spec.summary       = %q{HyperLogLog implementation in pure Ruby}
+  spec.homepage      = "https://github.com/alindeman/hyperll"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 2.14"
+end

data/lib/hyperll.rb

@@ -0,0 +1,2 @@
+require "hyperll/version"
+require "hyperll/hyper_log_log"

data/lib/hyperll/hyper_log_log.rb

@@ -0,0 +1,86 @@
+require_relative 'register_set'
+require_relative 'murmur_hash'
+
+module Hyperll
+  class HyperLogLog
+    INT_SIZE = 32
+    INT_HASH = 0xFFFFFFFF
+
+    attr_reader :log2m
+
+    def self.unserialize(serialized)
+      log2m, rs_size, *rs_values = serialized.unpack("N*")
+      new(log2m, RegisterSet.new(2 ** log2m, rs_values))
+    end
+
+    # Constructs a new HyperLogLog instance
+    #
+    # log2m - accuracy of the counter; larger values are more accurate
+    def initialize(log2m, register_set = nil)
+      @log2m = log2m
+      @count = 2 ** log2m
+      @register_set = register_set || RegisterSet.new(@count)
+
+      case log2m
+      when 4
+        @alphaMM = 0.673 * @count * @count
+      when 5
+        @alphaMM = 0.697 * @count * @count
+      when 6
+        @alphaMM = 0.709 * @count * @count
+      else
+        @alphaMM = (0.7213 / (1 + 1.079 / @count)) * @count * @count
+      end
+    end
+
+    def offer(obj)
+      offer_hashed(MurmurHash.hash(obj))
+    end
+
+    def offer_hashed(value)
+      j = value >> (INT_SIZE - @log2m)
+      r = number_of_leading_zeros(((value << @log2m) & INT_HASH) | (1 << (@log2m - 1)) + 1) + 1
+      @register_set.update_if_greater(j, r)
+    end
+
+    def cardinality
+      register_sum = 0.0
+      zeros = 0.0
+      @register_set.each do |value|
+        register_sum += 1.0 / (1 << value)
+        zeros += 1 if value == 0
+      end
+
+      estimate = @alphaMM * (1 / register_sum)
+      if estimate <= (5.0 / 2.0) * @count
+        # small range estimate
+        (@count * Math.log(@count / zeros)).round
+      else
+        estimate.round
+      end
+    end
+
+    def merge(*others)
+      raise "Cannot merge hyperloglogs of different sizes" unless others.all? { |o| o.log2m == log2m }
+
+      others.each do |other|
+        @register_set.merge(other.register_set)
+      end
+
+      self
+    end
+
+    def serialize
+      [@log2m, @register_set.size * 4].pack("N*") + @register_set.serialize
+    end
+
+    protected
+    def number_of_leading_zeros(int)
+      -(Math.log2(int).to_i - 31)
+    end
+
+    def register_set
+      @register_set
+    end
+  end
+end

data/lib/hyperll/murmur_hash.rb

@@ -0,0 +1,94 @@
+module Hyperll
+  # Adapted from <https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/hash/MurmurHash.java>
+  class MurmurHash
+    INT_MASK = 0xFFFFFFFF
+
+    def self.hash(obj)
+      if Integer === obj
+        hash_int(obj)
+      else
+        hash_string(obj.to_s)
+      end
+    end
+
+    def self.hash_int(data)
+      m = 0x5bd1e995
+      r = 24
+
+      h = 0
+
+      k = (data * m) & INT_MASK
+      k ^= k >> r
+      h ^= (k * m) & INT_MASK
+
+      k = ((data >> 32) * m) & INT_MASK
+      k ^= k >> r
+      h *= m
+      h ^= k * m
+      h &= INT_MASK
+
+      h ^= h >> 13
+      h *= m
+      h &= INT_MASK
+      h ^= h >> 15
+
+      h & INT_MASK
+    end
+    private_class_method :hash_int
+
+    def self.hash_string(str, seed = -1)
+      data = str.bytes
+      length = data.length
+
+      m = 0x5bd1e995
+      r = 24
+
+      h = seed ^ length
+      len_4 = length >> 2
+
+      0.upto(len_4 - 1) do |i|
+        i_4 = i << 2
+        k = data[i_4 + 3]
+        k = k << 8
+        k = k | (data[i_4 + 2] & 0xff)
+        k = k << 8
+        k = k | (data[i_4 + 1] & 0xff)
+        k = k << 8
+        k = k | (data[i_4 + 0] & 0xff)
+        k *= m
+        k &= INT_MASK
+        k ^= k >> r
+        k *= m
+        h *= m
+        h ^= k
+        h &= INT_MASK
+      end
+
+      len_m = len_4 << 2
+      left = length - len_m
+
+      if left != 0
+        if left >= 3
+          h ^= data[length - 3] << 16
+        end
+        if left >= 2
+          h ^= data[length - 2] << 8
+        end
+        if left >= 1
+          h ^= data[length - 1]
+        end
+
+        h *= m
+        h &= INT_MASK
+      end
+
+      h ^= h >> 13
+      h *= m
+      h &= INT_MASK
+      h ^= h >> 15
+
+      h
+    end
+    private_class_method :hash_string
+  end
+end

data/lib/hyperll/register_set.rb

@@ -0,0 +1,86 @@
+module Hyperll
+  class RegisterSet
+    include Enumerable
+
+    LOG2_BITS_PER_WORD = 6
+    REGISTER_SIZE = 5
+    INTEGER_SIZE = 32
+
+    attr_reader :count, :size
+
+    def initialize(count, values = nil)
+      @count = count
+
+      @bits = count / LOG2_BITS_PER_WORD
+      if @bits.zero?
+        @size = 1
+      elsif (@bits % INTEGER_SIZE).zero?
+        @size = @bits
+      else
+        @size = @bits + 1
+      end
+
+      @values = values || Array.new(@size, 0)
+    end
+
+    def []=(position, value)
+      bucket = position / LOG2_BITS_PER_WORD
+      shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD))
+
+      @values[bucket] = (@values[bucket] & ~(0x1f << shift)) | (value << shift)
+    end
+
+    def [](position)
+      bucket = position / LOG2_BITS_PER_WORD
+      shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD))
+
+      return (@values[bucket] & (0x1f << shift)) >> shift
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+      @count.times do |i|
+        yield self[i]
+      end
+    end
+
+    def update_if_greater(position, value)
+      bucket = position / LOG2_BITS_PER_WORD
+      shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD));
+      mask = 0x1f << shift;
+
+      current_value = @values[bucket] & mask
+      new_value = value << shift
+      if current_value < new_value
+        @values[bucket] = (@values[bucket] & ~mask) | new_value
+        true
+      else
+        false
+      end
+    end
+
+    def merge(other)
+      @size.times do |bucket|
+        word = 0
+        LOG2_BITS_PER_WORD.times do |j|
+          mask = 0x1f << (REGISTER_SIZE * j);
+
+          this_val = self.values[bucket] & mask
+          other_val = other.values[bucket] & mask
+          word |= [this_val, other_val].max
+        end
+
+        @values[bucket] = word
+      end
+    end
+
+    def serialize
+      @values.pack("N*")
+    end
+
+    protected
+    def values
+      @values
+    end
+  end
+end

data/lib/hyperll/version.rb

@@ -0,0 +1,3 @@
+module Hyperll
+  VERSION = "0.0.1"
+end

data/spec/hyperll/hyper_log_log_spec.rb

@@ -0,0 +1,70 @@
+require 'hyperll/hyper_log_log'
+
+module Hyperll
+  describe HyperLogLog do
+    it 'computes cardinality' do
+      hll = HyperLogLog.new(16)
+      hll.offer(0)
+      hll.offer(1)
+      hll.offer(2)
+      hll.offer(3)
+      hll.offer(16)
+      hll.offer(17)
+      hll.offer(18)
+      hll.offer(19)
+      hll.offer(19)
+
+      expect(hll.cardinality).to eq(8)
+    end
+
+    it 'is accurate within an expected amount for high cardinalities' do
+      hll = HyperLogLog.new(10)
+
+      size = 1_000_000
+      size.times do
+        hll.offer(rand(2**63))
+      end
+
+      expect(hll.cardinality).to be_within(10).percent_of(size)
+    end
+
+    it 'merges with other hyperloglog instances' do
+      size = 100_000
+      hlls = Array.new(5) do
+        HyperLogLog.new(16).tap { |hll|
+          size.times { hll.offer(rand(2**63)) }
+        }
+      end
+
+      merged = HyperLogLog.new(16)
+      merged.merge(*hlls)
+
+      expect(merged.cardinality).to be_within(10).percent_of(size * hlls.length)
+    end
+
+    it 'serializes to a string' do
+      hll = HyperLogLog.new(4)
+      hll.offer(1)
+      hll.offer(2)
+
+      # h = Java::com::clearspring::analytics::stream::cardinality::HyperLogLog.new(4)
+      # h.offer(1)
+      # h.offer(2)
+      # h.getBytes()
+      expect(hll.serialize.unpack("C*")).to eq(
+        [0, 0, 0, 4, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0]
+      )
+    end
+
+    it 'unserializes from a string' do
+      serialized = [0, 0, 0, 4, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0].pack("C*")
+      hll = HyperLogLog.unserialize(serialized)
+
+      expect(hll.cardinality).to eq(2)
+      hll.offer(1)
+      hll.offer(2)
+      hll.offer(3)
+      expect(hll.cardinality).to eq(3)
+    end
+  end
+end

data/spec/hyperll/murmur_hash_spec.rb

@@ -0,0 +1,25 @@
+require 'hyperll/murmur_hash'
+
+module Hyperll
+  describe MurmurHash do
+    it 'hashes integers' do
+      # java.lang.Integer.toHexString(Java::com::clearspring::analytics::hash::MurmurHash.hash(1))
+      expect(MurmurHash.hash(1)).to eq("5b04c018".to_i(16))
+      expect(MurmurHash.hash(2)).to eq("86e25492".to_i(16))
+      expect(MurmurHash.hash(1000)).to eq("a373b8db".to_i(16))
+      expect(MurmurHash.hash(5000)).to eq("5e1abaac".to_i(16))
+      expect(MurmurHash.hash(18_000_000)).to eq("347b61c7".to_i(16))
+    end
+
+    it 'hashes integers larger than 32 bits' do
+      expect(MurmurHash.hash(2 ** 33)).to eq("ab332279".to_i(16))
+      expect(MurmurHash.hash((2 ** 36) - 1)).to eq("db264be3".to_i(16))
+    end
+
+    it 'hashes strings' do
+      # java.lang.Integer.toHexString(Java::com::clearspring::analytics::hash::MurmurHash.hash("abc123"))
+      expect(MurmurHash.hash("abc123")).to eq("375301eb".to_i(16))
+      expect(MurmurHash.hash("The quick brown fox jumped over the lazy dog")).to eq("fe639b68".to_i(16))
+    end
+  end
+end

data/spec/hyperll/register_set_spec.rb

@@ -0,0 +1,71 @@
+require 'hyperll/register_set'
+
+module Hyperll
+  describe RegisterSet do
+    it "retrieves previously set values" do
+      rs = RegisterSet.new(2 ** 4)
+
+      rs[0] = 11
+      expect(rs[0]).to eq(11)
+    end
+
+    it "retrieves previously set values for small bits" do
+      rs = RegisterSet.new(6)
+
+      rs[0] = 11
+      expect(rs[0]).to eq(11)
+    end
+
+    it "merges with other register sets" do
+      rand = Random.new(2)
+      count = 32
+      rs = RegisterSet.new(count)
+
+      rss = []
+      5.times do |i|
+        rss[i] = RegisterSet.new(count)
+        count.times do |pos|
+          val = rand.rand(10)
+          rs.update_if_greater(pos, val)
+          rss[i][pos] = val
+        end
+      end
+
+      merged = RegisterSet.new(count)
+      rss.each do |set|
+        merged.merge(set)
+      end
+
+      rs.each_with_index do |value, index|
+        expect(value).to eq(merged[index])
+      end
+    end
+
+    it "merges with other register sets using update_if_greater" do
+      rand = Random.new(2)
+      count = 32
+      rs = RegisterSet.new(count)
+
+      rss = []
+      5.times do |i|
+        rss[i] = RegisterSet.new(count)
+        count.times do |pos|
+          val = rand.rand(10)
+          rs.update_if_greater(pos, val)
+          rss[i][pos] = val
+        end
+      end
+
+      merged = RegisterSet.new(count)
+      rss.each do |set|
+        set.each_with_index do |value, index|
+          merged.update_if_greater(index, value)
+        end
+      end
+
+      rs.each_with_index do |value, index|
+        expect(value).to eq(merged[index])
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: hyperll
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Andy Lindeman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+description: HyperLogLog implementation in pure Ruby
+email:
+- andy@andylindeman.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- hyperll.gemspec
+- lib/hyperll.rb
+- lib/hyperll/hyper_log_log.rb
+- lib/hyperll/murmur_hash.rb
+- lib/hyperll/register_set.rb
+- lib/hyperll/version.rb
+- spec/hyperll/hyper_log_log_spec.rb
+- spec/hyperll/murmur_hash_spec.rb
+- spec/hyperll/register_set_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/alindeman/hyperll
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: HyperLogLog implementation in pure Ruby
+test_files:
+- spec/hyperll/hyper_log_log_spec.rb
+- spec/hyperll/murmur_hash_spec.rb
+- spec/hyperll/register_set_spec.rb
+- spec/spec_helper.rb