tdigest 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb2b554c557253ae6826b504943a315bf4f660a0
-  data.tar.gz: 4f86fdef222ec8f3ddd7746ccfb7767cfbd1bd26
+  metadata.gz: d6b00ddf3a1a0b0a5989002fb3cf8fd51b352c6a
+  data.tar.gz: 60d1e7f4b42e3e38300f2f9380f9a91077bbd02f
 SHA512:
-  metadata.gz: c5b5567069b6958c551721f8c6c6d707e9a08f6f20c23473b39e732c853a92e4ef1e20406dba46d67ddf2b496ce94224501f6605def6f36be0b65b7f586aaa01
-  data.tar.gz: c00dfc8c3bb483793c5f5b2c8a4ea8ad9b324c4ce506a7930db362d31f42e2f1759215552d25140ab5667be969e62ee79fcc4e72cf7dcb0ca976d0fbae8b95d3
+  metadata.gz: d7154db5857ee2b184ff16c1f2afb7ea110be15cb1d4253e5287b0bf6e656232fccdffecf1f0a3ba6f52626039f0498b030a56cc5e71b51f0a6c82c60f15c7f2
+  data.tar.gz: 7265929cc8f77b8973cd9d60240c38f3ae3ce29e9af6644e18041c7a56f72ec5980bccf1696cb232865d294853821ce31d8582e57bd26806f79785610b1aa3d9

data/.ruby-version

@@ -0,0 +1 @@
+2.2.3

data/.travis.yml

@@ -1,4 +1,6 @@
 language: ruby
 rvm:
+  - 1.9.3
+  - 2.1.0
   - 2.2.3
 before_install: gem install bundler -v 1.10.6

data/README.md

@@ -1,6 +1,8 @@
 # Tdigest
 
 [![Gem Version](https://badge.fury.io/rb/tdigest.svg)](https://badge.fury.io/rb/tdigest)
+[![Build Status](https://travis-ci.org/castle/tdigest.svg?branch=master)](https://travis-ci.org/castle/tdigest)
+[![Coverage Status](https://coveralls.io/repos/castle/tdigest/badge.svg?branch=master&service=github)](https://coveralls.io/github/castle/tdigest?branch=master)
 
 Ruby implementation of Ted Dunning's [t-digest](https://github.com/tdunning/t-digest) data structure.
 
@@ -33,6 +35,36 @@ puts td.percentile(0.5)
 puts td.p_rank(0.95)
 ```
 
+#### Serialization
+
+This gem offers the same serialization options as the original [Java implementation](https://github.com/tdunning/t-digest). You can read more about T-digest persistance in [Chapter 3 in the paper](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf).
+
+**Standard encoding**
+
+This encoding uses 8-byte Double for the means and a 4-byte integers for counts.
+Size per centroid is a fixed 12-bytes.
+
+```ruby
+bytes = tdigest.as_bytes
+```
+
+**Compressed encoding**
+
+This encoding uses delta encoding with 4-byte floats for the means and variable
+length encoding for the counts. Size per centroid is between 5-12 bytes.
+
+```ruby
+bytes = tdigest.as_small_bytes
+```
+
+**Deserializing**
+
+Deserialization will automatically detect compression format
+
+```ruby
+ tdigest = TDigest::TDigest.from_bytes(bytes)
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/tdigest/centroid.rb

@@ -6,5 +6,9 @@ module TDigest
         send("#{p}=", value)
       end
     end
+
+    def as_json(_ = nil)
+      { m: mean, n: n }
+    end
   end
-end
+end

data/lib/tdigest/tdigest.rb

@@ -3,6 +3,9 @@ require 'tdigest/centroid'
 
 module TDigest
   class TDigest
+    VERBOSE_ENCODING = 1
+    SMALL_ENCODING   = 2
+
     attr_accessor :centroids
     def initialize(delta = 0.01, k = 25, cx = 1.1)
       @delta = delta
@@ -13,6 +16,45 @@ module TDigest
       reset!
     end
 
+    def as_bytes
+      # compression as defined by Java implementation
+      output = [VERBOSE_ENCODING, compression, size]
+      output += @centroids.map { |_, c| c.mean }
+      output += @centroids.map { |_, c| c.n }
+      output.pack("LdLd#{size}L#{size}")
+    end
+
+    def as_small_bytes
+      output = [SMALL_ENCODING, compression, size]
+      x = 0
+      # delta encoding allows saving 4-bytes floats
+      mean_arr = @centroids.map do |_, c|
+        val = c.mean - x
+        x = c.mean
+        val
+      end
+      output += mean_arr
+      # Variable length encoding of numbers
+      c_arr = @centroids.each_with_object([]) do |(_, c), arr|
+        k = 0
+        n = c.n
+        while n < 0 || n > 0x7f
+          b = 0x80 | (0x7f & n)
+          arr << b
+          n = n >> 7
+          k += 1
+          fail 'Unreasonable large number' if k > 6
+        end
+        arr << n
+      end
+      output += c_arr
+      output.pack("LdLf#{mean_arr.size}C#{c_arr.size}")
+    end
+
+    def as_json(_ = nil)
+      @centroids.map { |_, c| c.as_json }
+    end
+
     def bound_mean(x)
       upper = @centroids.upper_bound(x)
       lower = @centroids.lower_bound(x)
@@ -50,6 +92,10 @@ module TDigest
       nil
     end
 
+    def compression
+      1 / @delta
+    end
+
     def find_nearest(x)
       return nil if size == 0
 
@@ -149,33 +195,87 @@ module TDigest
       @centroids.map { |_, c| c }
     end
 
+    def self.from_bytes(bytes)
+      format, compression, size = bytes.unpack('LdL')
+      tdigest = new(1 / compression)
+
+      start_idx = 16 # after header
+      case format
+      when VERBOSE_ENCODING
+        array = bytes[start_idx..-1].unpack("d#{size}L#{size}")
+        means, counts = array.each_slice(size).to_a if array.size > 0
+      when SMALL_ENCODING
+        means = bytes[start_idx..(start_idx + 4 * size)].unpack("f#{size}")
+        # Decode delta encoding of means
+        x = 0
+        means.map! do |m|
+          m += x
+          x = m
+          m
+        end
+        counts_bytes = bytes[(start_idx + 4 * size)..-1].unpack('C*')
+        counts = []
+        # Decode variable length integer bytes
+        size.times do
+          v = counts_bytes.shift
+          z = 0x7f & v
+          shift = 7
+          while (v & 0x80) != 0
+            fail 'Shift too large in decode' if shift > 28
+            v = counts_bytes.shift || 0
+            z += (v & 0x7f) << shift
+            shift += 7
+          end
+          counts << z
+        end
+        # This shouldn't happen
+        fail 'Mismatch' unless counts.size == means.size
+      else
+        fail 'Unknown compression format'
+      end
+      if means && counts
+        means.zip(counts).each { |val| tdigest.push(val[0], val[1]) }
+      end
+      tdigest
+    end
 
-    private
+    def self.from_json(array)
+      tdigest = new
+      # Handle both string and symbol keys
+      array.each { |a| tdigest.push(a['m'] || a[:m], a['n'] || a[:n]) }
+      tdigest
+    end
 
+    private
 
     def _add_weight(nearest, x, n)
       unless x == nearest.mean
         nearest.mean += n * (x - nearest.mean) / (nearest.n + n)
       end
 
+      _cumulate(false, true) if nearest.mean_cumn.nil?
+
       nearest.cumn += n
-      nearest.mean_cumn += n / 2
+      nearest.mean_cumn += n / 2.0
       nearest.n += n
       @n += n
 
       nil
     end
 
-    def _cumulate(exact = false)
-      factor = @last_cumulate == 0 ? Float::INFINITY : (@n / @last_cumulate)
-      if @n == @last_cumulate ||
-        !exact && @cx && @cx > (factor)
-        return
+    def _cumulate(exact = false, force = false)
+      unless force
+        factor = if @last_cumulate == 0
+          Float::INFINITY
+        else
+          (@n.to_f / @last_cumulate)
+        end
+        return if @n == @last_cumulate || (!exact && @cx && @cx > (factor))
       end
 
       cumn = 0
       @centroids.each do |_, c|
-        c.mean_cumn = cumn + c.n / 2
+        c.mean_cumn = cumn + c.n / 2.0
         cumn = c.cumn = cumn + c.n
       end
       @n = @last_cumulate = cumn

data/lib/tdigest/version.rb

@@ -1,3 +1,3 @@
 module TDigest
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/tdigest.gemspec

@@ -21,7 +21,9 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'rbtree', '~> 0.4.2'
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest"
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'minitest', '~> 5.8.3'
+  spec.add_development_dependency 'coveralls', '~> 0.8.10'
+  spec.add_development_dependency 'simplecov', '~> 0.11.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tdigest
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Sebastian Wallin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-12-07 00:00:00.000000000 Z
+date: 2016-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbtree
@@ -56,16 +56,44 @@ dependencies:
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.8.3
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.10
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.10
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.11.1
 description: Ruby implementation of Dunning's T-Digest for streaming quantile approximation
 email:
 - sebastian.wallin@gmail.com
@@ -74,6 +102,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - ".travis.yml"
 - Gemfile
 - LICENSE.txt