RubyGems - quantile_estimator - Versions diffs - 0.0.2 - Mend

quantile_estimator 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (46) hide show

checksums.yaml +7 -0
data/.ruby-gemset +1 -0
data/.ruby-version +1 -0
data/Gemfile +4 -0
data/Gemfile.lock +17 -0
data/LICENSE.txt +22 -0
data/README.md +85 -0
data/Rakefile +9 -0
data/benchmark.rb +21 -0
data/doc/Cursor.html +422 -0
data/doc/Estimator.html +779 -0
data/doc/Invariant.html +115 -0
data/doc/Invariant/Biased.html +268 -0
data/doc/Invariant/Invariant.html +193 -0
data/doc/Invariant/SingleTarget.html +278 -0
data/doc/Invariant/Targeted.html +278 -0
data/doc/Item.html +620 -0
data/doc/Quantile.html +270 -0
data/doc/QuantileEstimator.html +117 -0
data/doc/_index.html +211 -0
data/doc/class_list.html +54 -0
data/doc/compression.png +0 -0
data/doc/css/common.css +1 -0
data/doc/css/full_list.css +57 -0
data/doc/css/style.css +338 -0
data/doc/file.README.html +186 -0
data/doc/file_list.html +56 -0
data/doc/frames.html +26 -0
data/doc/index.html +186 -0
data/doc/js/app.js +219 -0
data/doc/js/full_list.js +178 -0
data/doc/js/jquery.js +4 -0
data/doc/method_list.html +221 -0
data/doc/time.png +0 -0
data/doc/top-level-namespace.html +114 -0
data/lib/estimator.rb +120 -0
data/lib/quantile_estimator/cursor.rb +24 -0
data/lib/quantile_estimator/invariant.rb +47 -0
data/lib/quantile_estimator/item.rb +21 -0
data/lib/quantile_estimator/quantile.rb +3 -0
data/lib/quantile_estimator/test.rb +37 -0
data/lib/quantile_estimator/version.rb +3 -0
data/pkg/quantile_estimator-0.0.1.gem +0 -0
data/quantile_estimator.gemspec +29 -0
data/test/test_quantile_estimator.rb +85 -0
metadata +120 -0

data/doc/time.png ADDED Viewed

Binary file

data/doc/top-level-namespace.html ADDED Viewed

@@ -0,0 +1,114 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.8.7.3
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  hasFrames = window.top.frames.main ? true : false;
+  relpath = '';
+  framesUrl = "frames.html#!" + escape(window.location.href);
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div id="header">
+      <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+  <div class="noframes"><span class="title">(</span><a href="." target="_top">no frames</a><span class="title">)</span></div>
+</div>
+      <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+      Class List
+    </a>
+    <a class="full_list_link" id="method_list_link"
+        href="method_list.html">
+      Method List
+    </a>
+    <a class="full_list_link" id="file_list_link"
+        href="file_list.html">
+      File List
+    </a>
+</div>
+      <div class="clear"></div>
+    </div>
+    <iframe id="search_frame"></iframe>
+    <div id="content"><h1>Top Level Namespace
+</h1>
+<dl class="box">
+</dl>
+<div class="clear"></div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Invariant.html" title="Invariant (module)">Invariant</a></span>, <span class='object_link'><a href="QuantileEstimator.html" title="QuantileEstimator (module)">QuantileEstimator</a></span>
+      <strong class="classes">Classes:</strong> <span class='object_link'><a href="Cursor.html" title="Cursor (class)">Cursor</a></span>, <span class='object_link'><a href="Estimator.html" title="Estimator (class)">Estimator</a></span>, <span class='object_link'><a href="Item.html" title="Item (class)">Item</a></span>, <span class='object_link'><a href="Quantile.html" title="Quantile (class)">Quantile</a></span>
+</p>
+</div>
+    <div id="footer">
+  Generated on Fri Nov 15 15:39:43 2013 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.8.7.3 (ruby-2.0.0).
+</div>
+  </body>
+</html>

data/lib/estimator.rb ADDED Viewed

@@ -0,0 +1,120 @@
+require 'quantile_estimator/invariant'
+require 'quantile_estimator/item'
+require 'quantile_estimator/cursor'
+class Estimator
+  attr_accessor :samples
+  attr_accessor :n
+  attr_reader :invariant
+  # Creates a new quantile Estimator object using the provided invariant.
+  #
+  # == Parameters:
+  # value::
+  #   An Invariant object
+  #
+  def initialize(invariant)
+    @invariant = invariant
+    self.samples = []
+    self.n = 0
+  end
+  # Inserts a new element into the quantile estimator.
+  # O(n), where n is the number of elements of the internal data structure
+  #
+  # == Parameters:
+  # value::
+  #   A Fixnum to observe
+  #
+  # == Returns:
+  # The number of observations after the insertion
+  #
+  def insert(value)
+    i = 0
+    r_i = 0
+    while(i < samples.length)
+      item = samples[i]
+      break if item.value > value # determines the order
+      r_i = r_i + item.g
+      i += 1
+    end
+    delta = if (i-1 < 0) || (i == samples.length)
+              0
+            else
+              # r_i
+              [0, invariant.upper_bound(r_i, n).floor - 1].max
+            end
+    samples.insert(i, Item.new(value, 1, delta, r_i))
+    while(i < samples.length)
+      item = samples[i]
+      r_i = r_i + item.g
+      item.rank = r_i
+      i += 1
+    end
+    self.n += 1
+  end
+  # Compresses the internal data-structure.
+  # O(n), where n is the number of elements of the internal data structure
+  #
+  # == Parameters:
+  #
+  # == Returns:
+  #  The new size of the data-structure
+  def compress!
+    c = Cursor.new(self.samples, self.samples.length - 1)
+    while (~c != nil) && (~c.previous != nil)
+      if ((~c.previous).g + (~c).g + (~c).delta).to_f <=
+          invariant.upper_bound((~c.previous).rank, n)
+        removed   = ~c.previous
+        (~c).rank = removed.rank
+        (~c).g   += removed.g
+        c.previous.remove!
+        c = c.previous
+      end
+      c = c.previous
+    end
+    self.samples.length
+  end
+  # Queries de estimator for the given rank.
+  # O(n), where n is the number of elements of the internal data structure
+  #
+  # == Parameters:
+  # phi::
+  #
+  # A Fixnum between (0, 1) representing the rank to be queried (i.e, 0.5 represents
+  # the 50% quantile)
+  #
+  # == Returns:
+  # The approximate value for the quantile you are checking
+  #
+  def query(phi)
+    if n == 0
+      nil
+    else
+      rank = 0
+      c = Cursor.new(samples)
+      phi_n = phi * n
+      last = (~c).value
+      while ~c != nil
+        last = (~c).value
+        break if ~c.next == nil
+        c = c.next
+        if (rank + (~c).g + (~c).delta) > (phi_n + (invariant.upper_bound(phi_n, n) / 2))
+          return last
+        end
+        rank += (~c).g
+      end
+      return (~c).value
+    end
+  end
+end

data/lib/quantile_estimator/cursor.rb ADDED Viewed

@@ -0,0 +1,24 @@
+class Cursor
+  def initialize(array, start=0)
+    @array = array
+    @start = start
+  end
+  def ~
+    if @start >= 0
+      @array[@start]
+    end
+  end
+  def remove!
+    @array.delete_at(@start)
+  end
+  def next
+    Cursor.new(@array, @start + 1)
+  end
+  def previous
+    Cursor.new(@array, @start - 1)
+  end
+end

data/lib/quantile_estimator/invariant.rb ADDED Viewed

@@ -0,0 +1,47 @@
+module Invariant
+  class Invariant
+    def upper_bound(rank, n)
+      raise "Implement me"
+    end
+  end
+  class Biased < Invariant
+    def initialize(epsilon)
+      @epsilon = epsilon
+    end
+    def upper_bound(rank, n)
+      2 * @epsilon * rank
+    end
+  end
+  class SingleTarget < Invariant
+    def initialize(phi, epsilon)
+      @phi = phi
+      @epsilon = epsilon
+    end
+    def upper_bound(rank, n)
+      if @phi * n <= rank
+        (2 * @epsilon * rank) / @phi
+      else
+        (2 * @epsilon * (n - rank)) / (1 - @phi)
+      end
+    end
+  end
+  class Targeted < Invariant
+    def initialize(target_values)
+      @targets = target_values.map { |target_value|
+        phi, epsilon = target_value
+        SingleTarget.new(phi, epsilon)
+      }
+    end
+    def upper_bound(rank, n)
+      @targets.map { |target|
+        target.upper_bound(rank, n)
+      }.min
+    end
+  end
+end

data/lib/quantile_estimator/item.rb ADDED Viewed

@@ -0,0 +1,21 @@
+# n = items
+# k = asymptote
+# S = data structure containing multiple tuples
+# s = samples of items from the data stream
+class Item
+  attr_accessor :value, :g, :delta, :rank
+  def initialize(value, g, delta, rank=nil)
+    self.value = value
+    self.g = g
+    self.delta = delta
+    self.rank  = rank
+  end
+  def merge(item)
+    Item.new(item.value, self.g + item.g, item.delta, item.rank)
+  end
+  def to_s
+    [value, g, delta, rank].inspect
+  end
+end

data/lib/quantile_estimator/quantile.rb ADDED Viewed

@@ -0,0 +1,3 @@
+class Quantile
+  attr_accessor :quantile, :error
+end

data/lib/quantile_estimator/test.rb ADDED Viewed

@@ -0,0 +1,37 @@
+require "rubygems"
+require "quantile_estimator"
+require "item"
+require "invariant"
+# quantile = Quantile.new
+# quantile.quantile, quantile.error = 0.90, 0.001
+invariant = BiasedInvariant.new(0.00001)
+quantile_estimator = QuantileEstimator.new(invariant)
+# (1..1000).each do |x|
+#   # quantile_estimator.insert((1..1000).to_a[(rand * 100).floor])
+#   quantile_estimator.insert(rand)
+#   if x % 50 == 1
+#     p "size #{quantile_estimator.samples.size}"
+#     p "query(0.495) #{quantile_estimator.query(0.495)}"
+#     quantile_estimator.compress!
+#   end
+# end
+# p (1..10).to_a.shuffle
+test_values = [1, 4, 6, 5, 3, 10, 8, 9, 2, 7]
+test_values.each do |x|
+  quantile_estimator.insert(x)
+end
+quantile_estimator.samples.each {|x| p x}
+# p "size #{quantile_estimator.samples.size}"
+p "query(0.495) #{quantile_estimator.query(0.495)}"
+# p quantile_estimator.samples
+# quantile_estimator.compress!
+# p "size #{quantile_estimator.samples.size}"
+# p "query(0.495) #{quantile_estimator.query(0.495)}"

data/lib/quantile_estimator/version.rb ADDED Viewed

@@ -0,0 +1,3 @@
+module QuantileEstimator
+  VERSION = "0.0.2"
+end

data/pkg/quantile_estimator-0.0.1.gem ADDED Viewed

Binary file

data/quantile_estimator.gemspec ADDED Viewed

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'quantile_estimator/version'
+Gem::Specification.new do |spec|
+  spec.name          = "quantile_estimator"
+  spec.version       = QuantileEstimator::VERSION
+  spec.authors       = ["Diego Echeverri"]
+  spec.email         = ["diegoeche@gmail.com"]
+  spec.description   = %q{
+    Implementation of quantile estimators based on.
+    Cormode et. al.: "Effective Computation of Biased Quantiles over Data Streams"
+  }
+  spec.summary       = %q{
+    This gem implements a simple quantile estimator using Ruby Arrays.
+  }
+  spec.homepage      = "https://github.com/diegoeche/quantile_estimator.rb"
+  spec.license       = "MIT"
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib", "lib/quantile_estimator"]
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/test/test_quantile_estimator.rb ADDED Viewed

@@ -0,0 +1,85 @@
+require 'test/unit'
+require 'estimator'
+# The current tests come from the implementation at
+# https://github.com/odo/quantile_estimator
+class QuantileEstimatorTest < Test::Unit::TestCase
+  def test_biased_with_uniform_distribution
+    invariant = Invariant::Biased.new(0.00001)
+    estimator = Estimator.new(invariant)
+    # Equivalent to: (1..10).to_a.shuffle
+    test_values = [1, 4, 6, 5, 3, 10, 8, 9, 2, 7]
+    test_values.each do |x|
+      estimator.insert(x)
+    end
+    assert_equal 10, estimator.n
+    assert_equal 5, estimator.query(0.45)
+    # In this case it should not compress
+    estimator.compress!
+    assert_equal 10, estimator.n
+    assert_equal 5, estimator.query(0.45)
+    assert_equal 3, estimator.query(0.21)
+  end
+  def test_compression
+    invariant = Invariant::Biased.new(0.2)
+    estimator = Estimator.new(invariant)
+    # Equivalent to: (1..10).to_a.shuffle
+    test_values = [1, 4, 6, 5, 3, 10, 8, 9, 2, 7]
+    test_values.each do |x|
+      estimator.insert(x)
+    end
+    assert_equal 10, estimator.n
+    assert_equal 6, estimator.query(0.45)
+    estimator.compress!
+    assert_equal 8, estimator.samples.length
+    assert_equal 6, estimator.query(0.45)
+    assert_equal 10, estimator.query(0.8)
+  end
+  def test_targeted_invariant
+    invariant = Invariant::Targeted.new([
+                                         [0.05, 0.02],
+                                         [0.5,  0.02],
+                                         [0.95, 0.02]
+                                        ])
+    estimator = Estimator.new(invariant)
+    # Equivalent to: (1..10).to_a.shuffle
+    test_values = [10, 29, 27, 17, 20,
+                   6,  21, 13, 14, 2,
+                   16, 8,  3,  9,  5,
+                   7,  22, 12, 4,  11,
+                   26, 18, 25, 28, 19,
+                   30, 1,  23, 15, 24]
+    test_values.each do |x|
+      estimator.insert(x)
+    end
+    assert_equal 30, estimator.n
+    assert_equal 30, estimator.samples.size
+    assert_equal 15, estimator.query(0.45)
+    estimator.compress!
+    assert_equal 30, estimator.n
+    assert_equal 26, estimator.samples.length
+    assert_equal 16, estimator.query(0.5)
+    assert_equal 30, estimator.query(0.95)
+    assert_equal 2,  estimator.query(0.05)
+  end
+end