hyperll 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 02964ee6a649388f5d8cfb2f29c5b500b4f99894
-  data.tar.gz: 19f810ece27f49fcfe9f5eee45a47a3639622d18
+  metadata.gz: 82af95ec5cf82a356d3b5c1252c089509fa08193
+  data.tar.gz: d46c3121f220ad661fa1c4a6a6abaf3e31800726
 SHA512:
-  metadata.gz: 3a7941b3d6d866857e97951e19e69c398995efe884f8f950033218d886acc6502c874394a165fe327a68b70a208bb7c3a7ca1414c8a94ff1f81309324ec4ea39
-  data.tar.gz: b8edf1d57aec35bcb5c86f6468a71cd93a43c939e4cd2d33f26272ea1dc0b09347e99b95f7830b8f974e11b92fed86be58da3a865eae6ee39374dc7315e405b2
+  metadata.gz: 91892e69eb09d3c41450b32e8f52112597190dd1f0664d762e477d33e7506ecac40b519d2f541d4c965402885839f122fab75eb40f4466bd273002bc7fe84da7
+  data.tar.gz: 0858d530e8520ee1765edccbac24c71d33777a24dacea7a19a1428a7eef54f2e3640e9ffba578874cba619d2bcd2375f9d41e655ea3cefce61c4511684a9b662

data/lib/hyperll/delta_bytes.rb

@@ -3,6 +3,15 @@ require_relative 'varint'
 module Hyperll
   class DeltaBytes
     def self.compress(bytes)
+      compressed = Varint.write_unsigned_var_int(bytes.length)
+      previous_value = 0
+
+      bytes.each do |b|
+        compressed.concat(Varint.write_unsigned_var_int(b - previous_value))
+        previous_value = b
+      end
+
+      compressed
     end
 
     def self.uncompress(bytes)

data/lib/hyperll/hyper_log_log_plus.rb

@@ -186,6 +186,25 @@ module Hyperll
       self
     end
 
+    def serialize
+      str = ""
+      str << [-2].pack("N") # -VERSION
+      str << Varint.write_unsigned_var_int(p).pack("C*")
+      str << Varint.write_unsigned_var_int(sp).pack("C*")
+
+      case format
+      when :normal
+        str << Varint.write_unsigned_var_int(0).pack("C*")
+        str << Varint.write_unsigned_var_int(@register_set.size * 4).pack("C*")
+        str << @register_set.serialize
+      when :sparse
+        str << Varint.write_unsigned_var_int(1).pack("C*")
+        str << DeltaBytes.compress(@sparse_set).pack("C*")
+      end
+
+      str
+    end
+
     protected
     def sparse_set
       @sparse_set

data/lib/hyperll/varint.rb

@@ -11,5 +11,16 @@ module Hyperll
 
       value | (b << i)
     end
+
+    def self.write_unsigned_var_int(value)
+      bytes = []
+      while (value & 0xFFFFFF80) != 0
+        bytes << ((value & 0x7F) | 0x80)
+        value >>= 7
+      end
+
+      bytes << (value & 0x7F)
+      bytes
+    end
   end
 end

data/lib/hyperll/version.rb

@@ -1,3 +1,3 @@
 module Hyperll
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/spec/hyperll/delta_bytes_spec.rb

@@ -3,8 +3,12 @@ require 'hyperll/delta_bytes'
 
 module Hyperll
   describe DeltaBytes do
-    it 'uncompressed bytes' do
+    it 'uncompresses bytes' do
       expect(DeltaBytes.uncompress([2, -46, 5, -64, 4])).to eq([722, 1298])
     end
+
+    it 'compresses bytes' do
+      expect(DeltaBytes.compress([722, 1298])).to eq([2, 256 - 46, 5, 256 - 64, 4])
+    end
   end
 end

data/spec/hyperll/hyper_log_log_plus_spec.rb

@@ -18,6 +18,38 @@ module Hyperll
       end
     end
 
+    describe 'serialization' do
+      it 'serializes a sparse format to a string' do
+        # TODO: We can't yet construct a HyperLogLogPlus from scratch, so we
+        # deserialize a HyperLogLogPlus from stream-lib, then serialize it
+        # back and make sure it matches.
+
+        # h = Java::com::clearspring::analytics::stream::cardinality::HyperLogLogPlus.new(4, 10)
+        # h.offer(1)
+        # h.offer(2)
+        # h.getBytes()
+        serialized = [-1, -1, -1, -2, 4, 10, 1, 2, -46, 5, -64, 4].pack("C*")
+
+        hllp = HyperLogLogPlus.unserialize(serialized)
+        expect(hllp.serialize).to eq(serialized)
+      end
+
+      it 'serializes a normal format to a string' do
+        # TODO: We can't yet construct a HyperLogLogPlus from scratch, so we
+        # deserialize a HyperLogLogPlus from stream-lib, then serialize it
+        # back and make sure it matches.
+
+        # h = Java::com::clearspring::analytics::stream::cardinality::HyperLogLogPlus.new(4)
+        # h.offer(1)
+        # h.offer(2)
+        # h.getBytes()
+        serialized = [-1, -1, -1, -2, 4, 0, 0, 12, 2, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0].pack("C*")
+
+        hllp = HyperLogLogPlus.unserialize(serialized)
+        expect(hllp.serialize).to eq(serialized)
+      end
+    end
+
     describe 'format' do
       it 'defaults to normal (non-sparse) format' do
         hllp = HyperLogLogPlus.new(11)

data/spec/hyperll/varint_spec.rb

@@ -8,5 +8,11 @@ module Hyperll
       expect(Varint.read_unsigned_var_int([0x81, 0x01])).to eq(0x81)
       expect(Varint.read_unsigned_var_int([0x81, 0x81, 0x01])).to eq(0x4081)
     end
+
+    it 'writes unsigned variable length integers' do
+      expect(Varint.write_unsigned_var_int(0x35)).to eq([0x35])
+      expect(Varint.write_unsigned_var_int(0x81)).to eq([0x81, 0x01])
+      expect(Varint.write_unsigned_var_int(0x4081)).to eq([0x81, 0x81, 0x01])
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hyperll
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Andy Lindeman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-25 00:00:00.000000000 Z
+date: 2013-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler