hyperll 0.2.6 → 0.3.0

data/spec/hyperll/delta_bytes_spec.rb

@@ -1,14 +1,16 @@
 require 'spec_helper'
-require 'hyperll/delta_bytes'
+require 'hyperll'
 
 module Hyperll
   describe DeltaBytes do
     it 'uncompresses bytes' do
-      expect(DeltaBytes.uncompress([2, -46, 5, -64, 4])).to eq([722, 1298])
+      expect(DeltaBytes.uncompress([-46, 5, -64, 4])).to eq([722, 1298])
+      expect(DeltaBytes.uncompress([210, 5, 192, 4, 254, 67])).to eq([722, 1298, 10000])
     end
 
     it 'compresses bytes' do
-      expect(DeltaBytes.compress([722, 1298])).to eq([2, 256 - 46, 5, 256 - 64, 4])
+      expect(DeltaBytes.compress([722, 1298])).to eq([256 - 46, 5, 256 - 64, 4])
+      expect(DeltaBytes.compress([722, 1298, 10000])).to eq([210, 5, 192, 4, 254, 67])
     end
   end
 end

data/spec/hyperll/hyper_log_log_plus_spec.rb

@@ -1,6 +1,7 @@
 require 'spec_helper'
 require 'base64'
-require 'hyperll/hyper_log_log_plus'
+require 'hyperll'
+require 'json'
 
 module Hyperll
   describe HyperLogLogPlus do
@@ -206,6 +207,22 @@ module Hyperll
         expect(hllp.cardinality).to eq(6)
       end
 
+      it 'merges and keeps the cardinality exact, handling elements that are common to both sets' do
+        # Serialization after offering [12, 13, 22, 34, 38, 40, 41, 46, 49]
+        hllp = HyperLogLogPlus.unserialize(Base64.decode64("/////gsQAQnMYsoMtgak9AGMiwK8VbKiAYmU0wPVwK38Dw=="))
+
+        # Serialization after offering [2, 6, 19, 29, 41, 48]
+        hllp2 = HyperLogLogPlus.unserialize(Base64.decode64("/////gsQAQbwdJz0Afq4AbSZAqxX4i4="))
+
+        expect(hllp.cardinality).to eq(9)
+        expect(hllp2.cardinality).to eq(6)
+
+        # The set intersection of hllp and hllp2 has one element, 41, so after
+        # merging the cardinality should be 14.
+        hllp.merge(hllp2)
+        expect(hllp.cardinality).to eq(14)
+      end
+
       it 'merges and keeps the cardinality exact' do
         hllp = HyperLogLogPlus.unserialize(Base64.decode64("/////gsQAQOwX+yBA7TzAw=="))
         hllp2 = HyperLogLogPlus.unserialize(Base64.decode64("/////gsQAQ7SKbociFqGigLUL9oagCWmC+IdlBqkE8g7jFiCnwE="))
@@ -270,5 +287,17 @@ module Hyperll
         expect(hllp.cardinality).to eq(8) # 3 + 3 = 8; that's how it goes with hll
       end
     end
+
+    context 'merging multiple at a time' do
+      it 'merges' do
+        hllp = HyperLogLogPlus.unserialize([-1, -1, -1, -2, 4, 10, 1, 1, -110, 10].pack("C*"))
+        hllp2 = HyperLogLogPlus.unserialize([-1, -1, -1, -2, 4, 10, 1, 1, -46, 5].pack("C*"))
+        hllp3 = HyperLogLogPlus.unserialize([-1, -1, -1, -2, 4, 10, 1, 1, -124, 6].pack("C*"))
+
+        hllp.merge(hllp2, hllp3)
+        expect(hllp.format).to eq(:sparse)
+        expect(hllp.cardinality).to eq(3) # 1 + 1 + 1 = 3
+      end
+    end
   end
 end

data/spec/hyperll/hyper_log_log_spec.rb

@@ -1,5 +1,5 @@
 require 'spec_helper'
-require 'hyperll/hyper_log_log'
+require 'hyperll'
 
 module Hyperll
   describe HyperLogLog do

data/spec/hyperll/register_set_spec.rb

@@ -1,5 +1,5 @@
 require 'spec_helper'
-require 'hyperll/register_set'
+require 'hyperll'
 
 module Hyperll
   describe RegisterSet do
@@ -68,5 +68,22 @@ module Hyperll
         expect(value).to eq(merged[index])
       end
     end
+
+    it "serializes to a string" do
+      rs = RegisterSet.new(10)
+      rs[0] = 2
+      rs[1] = 3
+      rs[2] = 4
+
+      expect(rs.serialize).to eq("\x00\x00\x10b\x00\x00\x00\x00".force_encoding("ASCII-8BIT"))
+    end
+
+    it "unserializes from a string" do
+      rs = RegisterSet.new(10, "\x00\x00\x10b\x00\x00\x00\x00".unpack("N*"))
+
+      expect(rs[0]).to eq(2)
+      expect(rs[1]).to eq(3)
+      expect(rs[2]).to eq(4)
+    end
   end
 end

data/spec/hyperll/varint_spec.rb

@@ -1,5 +1,5 @@
 require 'spec_helper'
-require 'hyperll/varint'
+require 'hyperll'
 
 module Hyperll
   describe Varint do
@@ -14,5 +14,9 @@ module Hyperll
       expect(Varint.write_unsigned_var_int(0x81)).to eq([0x81, 0x01])
       expect(Varint.write_unsigned_var_int(0x4081)).to eq([0x81, 0x81, 0x01])
     end
+
+    it 'treats negative integers as their twos complement unsigned representation' do
+      expect(Varint.write_unsigned_var_int(-4)).to eq([252, 255, 255, 255, 15])
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hyperll
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Andy Lindeman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-29 00:00:00.000000000 Z
+date: 2013-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,11 +52,26 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.1
 description: HyperLogLog implementation in pure Ruby
 email:
 - andy@andylindeman.com
 executables: []
-extensions: []
+extensions:
+- ext/hyperll/extconf.rb
 extra_rdoc_files: []
 files:
 - .gitignore
@@ -66,18 +81,27 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- ext/hyperll/delta_bytes.c
+- ext/hyperll/delta_bytes.h
+- ext/hyperll/extconf.rb
+- ext/hyperll/hyper_log_log_plus.c
+- ext/hyperll/hyperll.c
+- ext/hyperll/hyperll.h
+- ext/hyperll/register_set.c
+- ext/hyperll/register_set.h
+- ext/hyperll/sparse_set.c
+- ext/hyperll/sparse_set.h
+- ext/hyperll/varint.c
+- ext/hyperll/varint.h
 - hyperll.gemspec
 - lib/hyperll.rb
-- lib/hyperll/delta_bytes.rb
 - lib/hyperll/hyper_log_log.rb
 - lib/hyperll/hyper_log_log_plus.rb
 - lib/hyperll/murmur_hash.rb
-- lib/hyperll/register_set.rb
-- lib/hyperll/util.rb
-- lib/hyperll/varint.rb
 - lib/hyperll/version.rb
 - spec/fixtures/10000.txt
 - spec/fixtures/mega.json
+- spec/fixtures/merge-many-sets.json
 - spec/hyperll/delta_bytes_spec.rb
 - spec/hyperll/hyper_log_log_plus_spec.rb
 - spec/hyperll/hyper_log_log_spec.rb
@@ -114,6 +138,7 @@ summary: HyperLogLog implementation in pure Ruby
 test_files:
 - spec/fixtures/10000.txt
 - spec/fixtures/mega.json
+- spec/fixtures/merge-many-sets.json
 - spec/hyperll/delta_bytes_spec.rb
 - spec/hyperll/hyper_log_log_plus_spec.rb
 - spec/hyperll/hyper_log_log_spec.rb

data/lib/hyperll/delta_bytes.rb

@@ -1,32 +0,0 @@
-require_relative 'varint'
-
-module Hyperll
-  class DeltaBytes
-    def self.compress(bytes)
-      compressed = Varint.write_unsigned_var_int(bytes.length)
-      previous_value = 0
-
-      bytes.each do |b|
-        compressed.concat(Varint.write_unsigned_var_int(b - previous_value))
-        previous_value = b
-      end
-
-      compressed
-    end
-
-    def self.uncompress(bytes)
-      uncompressed = []
-      previous_value = 0
-
-      length = Varint.read_unsigned_var_int(bytes)
-      length.times do
-        next_value = Varint.read_unsigned_var_int(bytes)
-
-        uncompressed << next_value + previous_value
-        previous_value = uncompressed.last
-      end
-
-      uncompressed
-    end
-  end
-end

data/lib/hyperll/register_set.rb

@@ -1,90 +0,0 @@
-require_relative 'util'
-
-module Hyperll
-  class RegisterSet
-    include Enumerable
-    include Util
-
-    LOG2_BITS_PER_WORD = 6
-    REGISTER_SIZE = 5
-
-    attr_reader :count, :size
-
-    def initialize(count, values = nil)
-      @count = count
-
-      @bits = count / LOG2_BITS_PER_WORD
-      if @bits.zero?
-        @size = 1
-      elsif (@bits % INTEGER_SIZE).zero?
-        @size = @bits
-      else
-        @size = @bits + 1
-      end
-
-      @values = values || Array.new(@size, 0)
-    end
-
-    def []=(position, value)
-      bucket = position / LOG2_BITS_PER_WORD
-      shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD))
-
-      @values[bucket] = ((@values[bucket] & ~(0x1f * POWERS_OF_TWO[shift])) | (value * POWERS_OF_TWO[shift]))
-      @values[bucket] &= INT_MASK if @values[bucket] > INT_MASK
-    end
-
-    def [](position)
-      bucket = position / LOG2_BITS_PER_WORD
-      shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD))
-
-      return (@values[bucket] & (0x1f * POWERS_OF_TWO[shift])) / POWERS_OF_TWO[shift]
-    end
-
-    def each
-      return enum_for(:each) unless block_given?
-      @count.times do |i|
-        yield self[i]
-      end
-    end
-
-    def update_if_greater(position, value)
-      bucket = position / LOG2_BITS_PER_WORD
-      shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD));
-      mask = (0x1f * POWERS_OF_TWO[shift])
-
-      current_value = @values[bucket] & mask
-      new_value = value * POWERS_OF_TWO[shift]
-      if current_value < new_value
-        @values[bucket] = ((@values[bucket] & ~mask) | new_value)
-        @values[bucket] &= INT_MASK if @values[bucket] > INT_MASK
-        true
-      else
-        false
-      end
-    end
-
-    def merge(other)
-      @size.times do |bucket|
-        word = 0
-        LOG2_BITS_PER_WORD.times do |j|
-          mask = 0x1f << (REGISTER_SIZE * j);
-
-          this_val = self.values[bucket] & mask
-          other_val = other.values[bucket] & mask
-          word |= [this_val, other_val].max
-        end
-
-        @values[bucket] = word
-      end
-    end
-
-    def serialize
-      @values.pack("N*")
-    end
-
-    protected
-    def values
-      @values
-    end
-  end
-end

data/lib/hyperll/util.rb

@@ -1,39 +0,0 @@
-module Hyperll
-  module Util
-    INT_MASK = 0xFFFFFFFF
-    INTEGER_SIZE = 32
-
-    POWERS_OF_TWO = 0.upto(32).map { |i| 2**i }.freeze
-
-    def number_of_leading_zeros(x)
-      return 32 if x == 0
-
-      n = 0
-      if x <= 0x0000FFFF
-        n += 16
-        x *= POWERS_OF_TWO[16]
-      end
-
-      if x <= 0x00FFFFFF
-        n += 8;
-        x *= POWERS_OF_TWO[8]
-      end
-
-      if x <= 0x0FFFFFFF
-        n += 4
-        x *= POWERS_OF_TWO[4]
-      end
-
-      if x <= 0x3FFFFFFF
-        n += 2
-        x *= POWERS_OF_TWO[2]
-      end
-
-      if x <= 0x7FFFFFFF
-        n += 1
-      end
-
-      n
-    end
-  end
-end

data/lib/hyperll/varint.rb

@@ -1,26 +0,0 @@
-module Hyperll
-  class Varint
-    def self.read_unsigned_var_int(bytes)
-      value, i, b = 0, 0, 0
-      while (b = bytes.shift) & 0x80 != 0
-        value |= (b & 0x7F) << i
-
-        i += 7
-        raise "Variable length quantity is too long" if i > 35
-      end
-
-      value | (b << i)
-    end
-
-    def self.write_unsigned_var_int(value)
-      bytes = []
-      while (value & 0xFFFFFF80) != 0
-        bytes << ((value & 0x7F) | 0x80)
-        value >>= 7
-      end
-
-      bytes << (value & 0x7F)
-      bytes
-    end
-  end
-end