quantile_estimator 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d96954e453f9b1c4dab4c71a9a26e00b8f26d138
+  data.tar.gz: 6da4c7c657a29ea5993d535586b3e4ab54bf8b4b
+SHA512:
+  metadata.gz: e9b1452ae828b888742a75eb601bd049969d01501cd17410a55bd14463c8fc23456ffc4a21a37aa78f1acac41e95150697c18dd81d07459d8219baa7f1ea444d
+  data.tar.gz: 7885e33a2b2792f39ef81dc87a1fb4f90ca8934f401efa408c701f4596b52e37574b444f90c08f64f65afd9b8a2ba3302c1f474214977a4701ebb42ecab95603

data/.ruby-gemset

@@ -0,0 +1 @@
+quantile_estimator

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.0.0-p0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in quantile_estimator.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,17 @@
+PATH
+  remote: .
+  specs:
+    quantile_estimator (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (10.0.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  quantile_estimator!
+  rake

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Diego Echeverri
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,85 @@
+# QuantileEstimator.rb
+
+This implements the quantile estimator described in the paper:
+
+Cormode et. al.:
+"Effective Computation of Biased Quantiles over Data Streams"
+
+Given the different strategies to implement the algorithm I used the easiest one by
+using https://github.com/odo/quantile_estimator as reference for the implementation.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'quantile_estimator'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install quantile_estimator
+
+## Usage
+
+First initialize the estimator. Right now you can either have a `Biased` or
+`Targeted` invariants. The targeted invariant let you select the quantiles you are
+particularly interested and will yield higher compression rates.
+
+    biased = Invariant::Biased.new(0.004)
+    targeted = Invariant::Targeted.new([
+                                        [0.05, 0.02],
+                                        [0.5,  0.02],
+                                        [0.95, 0.02]
+                                       ])
+
+    estimator = Estimator.new(targeted)
+
+Insertion of data is as simple as:
+
+    estimator.insert(value)
+
+The insertion of data _doesn't_ automatically compress it. To compress the data just
+call:
+
+    estimator.compress!
+
+Using these primitives you can build wraps to compress on every nth insert as shown
+in
+[this file](https://github.com/diegoeche/quantile_estimator.rb/blob/master/benchmark.rb)
+
+## Pretty Graphs
+
+Using a targeted invariant to keep track of the 0.05, 0.5 and 0.95 quantiles, a
+uniform source of random values and compressing every 100 iterations. We get the
+following behavior regarding the size of the internal data structure:
+
+![compression rate (elements size, lower is better)](https://raw.github.com/diegoeche/quantile_estimator.rb/master/doc/compression.png
+ "Elements in the estimator. Lower is better")
+
+Running time behavior is not too bad. The following graph shows the cost of
+insertions in the estimator. The homogeneous layer of outlayers probably corresponds
+to the compression cycles, while the bottom line is the cost of compression-less
+insertions.
+
+![runtime behavior (ms, lower is better)](https://raw.github.com/diegoeche/quantile_estimator.rb/master/doc/time.png "Time in ms. Lower is better")
+
+Different distributions, different invariants setups will have different behaviors.
+
+Check your real data before using this!
+
+## Known issues
+
+The implementation is known not to be thread-safe, and little effort has been done to
+optimize it.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/benchmark.rb

@@ -0,0 +1,21 @@
+require 'estimator'
+
+biased = Invariant::Biased.new(0.004)
+targeted = Invariant::Targeted.new([
+                                    [0.05, 0.02],
+                                    [0.5,  0.02],
+                                    [0.95, 0.02]
+                                   ])
+
+# estimator = Estimator.new(invariant)
+
+estimator = Estimator.new(targeted)
+
+10000.times do |i|
+  nowish = Time.now.to_f
+  estimator.insert(rand)
+  if i % 100 == 99
+    estimator.compress!
+  end
+  puts [i, 1000 * (Time.now.to_f - nowish), estimator.samples.size].join("\t")
+end

data/doc/Cursor.html

@@ -0,0 +1,422 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<title>
+  Class: Cursor
+  
+    &mdash; Documentation by YARD 0.8.7.3
+  
+</title>
+
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+
+<script type="text/javascript" charset="utf-8">
+  hasFrames = window.top.frames.main ? true : false;
+  relpath = '';
+  framesUrl = "frames.html#!" + escape(window.location.href);
+</script>
+
+
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+
+
+  </head>
+  <body>
+    <div id="header">
+      <div id="menu">
+  
+    <a href="_index.html">Index (C)</a> &raquo;
+    
+    
+    <span class="title">Cursor</span>
+  
+
+  <div class="noframes"><span class="title">(</span><a href="." target="_top">no frames</a><span class="title">)</span></div>
+</div>
+
+      <div id="search">
+  
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+      Class List
+    </a>
+  
+    <a class="full_list_link" id="method_list_link"
+        href="method_list.html">
+      Method List
+    </a>
+  
+    <a class="full_list_link" id="file_list_link"
+        href="file_list.html">
+      File List
+    </a>
+  
+</div>
+      <div class="clear"></div>
+    </div>
+
+    <iframe id="search_frame"></iframe>
+
+    <div id="content"><h1>Class: Cursor
+  
+  
+  
+</h1>
+
+<dl class="box">
+  
+    <dt class="r1">Inherits:</dt>
+    <dd class="r1">
+      <span class="inheritName">Object</span>
+      
+        <ul class="fullTree">
+          <li>Object</li>
+          
+            <li class="next">Cursor</li>
+          
+        </ul>
+        <a href="#" class="inheritanceTree">show all</a>
+      
+      </dd>
+    
+  
+  
+    
+  
+    
+  
+  
+  
+    <dt class="r2 last">Defined in:</dt>
+    <dd class="r2 last">lib/quantile_estimator/cursor.rb</dd>
+  
+</dl>
+<div class="clear"></div>
+
+
+
+
+
+
+
+
+  
+    <h2>
+      Instance Method Summary
+      <small>(<a href="#" class="summary_toggle">collapse</a>)</small>
+    </h2>
+
+    <ul class="summary">
+      
+        <li class="public ">
+  <span class="summary_signature">
+    
+      <a href="#initialize-instance_method" title="#initialize (instance method)">- (Cursor) <strong>initialize</strong>(array, start = 0) </a>
+    
+
+    
+  </span>
+  
+  
+    <span class="note title constructor">constructor</span>
+  
+  
+  
+  
+  
+  
+
+  
+    <span class="summary_desc"><div class='inline'>
+<p>A new instance of Cursor.</p>
+</div></span>
+  
+</li>
+
+      
+        <li class="public ">
+  <span class="summary_signature">
+    
+      <a href="#next-instance_method" title="#next (instance method)">- (Object) <strong>next</strong> </a>
+    
+
+    
+  </span>
+  
+  
+  
+  
+  
+  
+  
+
+  
+    <span class="summary_desc"><div class='inline'></div></span>
+  
+</li>
+
+      
+        <li class="public ">
+  <span class="summary_signature">
+    
+      <a href="#previous-instance_method" title="#previous (instance method)">- (Object) <strong>previous</strong> </a>
+    
+
+    
+  </span>
+  
+  
+  
+  
+  
+  
+  
+
+  
+    <span class="summary_desc"><div class='inline'></div></span>
+  
+</li>
+
+      
+        <li class="public ">
+  <span class="summary_signature">
+    
+      <a href="#remove%21-instance_method" title="#remove! (instance method)">- (Object) <strong>remove!</strong> </a>
+    
+
+    
+  </span>
+  
+  
+  
+  
+  
+  
+  
+
+  
+    <span class="summary_desc"><div class='inline'></div></span>
+  
+</li>
+
+      
+        <li class="public ">
+  <span class="summary_signature">
+    
+      <a href="#%7E-instance_method" title="#~ (instance method)">- (Object) <strong>~</strong> </a>
+    
+
+    
+  </span>
+  
+  
+  
+  
+  
+  
+  
+
+  
+    <span class="summary_desc"><div class='inline'></div></span>
+  
+</li>
+
+      
+    </ul>
+  
+
+<div id="constructor_details" class="method_details_list">
+  <h2>Constructor Details</h2>
+  
+    <div class="method_details first">
+  <h3 class="signature first" id="initialize-instance_method">
+  
+    - (<tt><span class='object_link'><a href="" title="Cursor (class)">Cursor</a></span></tt>) <strong>initialize</strong>(array, start = 0) 
+  
+
+  
+
+  
+</h3><div class="docstring">
+  <div class="discussion">
+    
+<p>Returns a new instance of Cursor</p>
+
+
+  </div>
+</div>
+<div class="tags">
+  
+
+</div><table class="source_code">
+  <tr>
+    <td>
+      <pre class="lines">
+
+
+2
+3
+4
+5</pre>
+    </td>
+    <td>
+      <pre class="code"><span class="info file"># File 'lib/quantile_estimator/cursor.rb', line 2</span>
+
+<span class='kw'>def</span> <span class='id identifier rubyid_initialize'>initialize</span><span class='lparen'>(</span><span class='id identifier rubyid_array'>array</span><span class='comma'>,</span> <span class='id identifier rubyid_start'>start</span><span class='op'>=</span><span class='int'>0</span><span class='rparen'>)</span>
+  <span class='ivar'>@array</span> <span class='op'>=</span> <span class='id identifier rubyid_array'>array</span>
+  <span class='ivar'>@start</span> <span class='op'>=</span> <span class='id identifier rubyid_start'>start</span>
+<span class='kw'>end</span></pre>
+    </td>
+  </tr>
+</table>
+</div>
+  
+</div>
+
+
+  <div id="instance_method_details" class="method_details_list">
+    <h2>Instance Method Details</h2>
+
+    
+      <div class="method_details first">
+  <h3 class="signature first" id="next-instance_method">
+  
+    - (<tt>Object</tt>) <strong>next</strong> 
+  
+
+  
+
+  
+</h3><table class="source_code">
+  <tr>
+    <td>
+      <pre class="lines">
+
+
+17
+18
+19</pre>
+    </td>
+    <td>
+      <pre class="code"><span class="info file"># File 'lib/quantile_estimator/cursor.rb', line 17</span>
+
+<span class='kw'>def</span> <span class='kw'>next</span>
+  <span class='const'>Cursor</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span><span class='ivar'>@array</span><span class='comma'>,</span> <span class='ivar'>@start</span> <span class='op'>+</span> <span class='int'>1</span><span class='rparen'>)</span>
+<span class='kw'>end</span></pre>
+    </td>
+  </tr>
+</table>
+</div>
+    
+      <div class="method_details ">
+  <h3 class="signature " id="previous-instance_method">
+  
+    - (<tt>Object</tt>) <strong>previous</strong> 
+  
+
+  
+
+  
+</h3><table class="source_code">
+  <tr>
+    <td>
+      <pre class="lines">
+
+
+21
+22
+23</pre>
+    </td>
+    <td>
+      <pre class="code"><span class="info file"># File 'lib/quantile_estimator/cursor.rb', line 21</span>
+
+<span class='kw'>def</span> <span class='id identifier rubyid_previous'>previous</span>
+  <span class='const'>Cursor</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span><span class='ivar'>@array</span><span class='comma'>,</span> <span class='ivar'>@start</span> <span class='op'>-</span> <span class='int'>1</span><span class='rparen'>)</span>
+<span class='kw'>end</span></pre>
+    </td>
+  </tr>
+</table>
+</div>
+    
+      <div class="method_details ">
+  <h3 class="signature " id="remove!-instance_method">
+  
+    - (<tt>Object</tt>) <strong>remove!</strong> 
+  
+
+  
+
+  
+</h3><table class="source_code">
+  <tr>
+    <td>
+      <pre class="lines">
+
+
+13
+14
+15</pre>
+    </td>
+    <td>
+      <pre class="code"><span class="info file"># File 'lib/quantile_estimator/cursor.rb', line 13</span>
+
+<span class='kw'>def</span> <span class='id identifier rubyid_remove!'>remove!</span>
+  <span class='ivar'>@array</span><span class='period'>.</span><span class='id identifier rubyid_delete_at'>delete_at</span><span class='lparen'>(</span><span class='ivar'>@start</span><span class='rparen'>)</span>
+<span class='kw'>end</span></pre>
+    </td>
+  </tr>
+</table>
+</div>
+    
+      <div class="method_details ">
+  <h3 class="signature " id="~-instance_method">
+  
+    - (<tt>Object</tt>) <strong>~</strong> 
+  
+
+  
+
+  
+</h3><table class="source_code">
+  <tr>
+    <td>
+      <pre class="lines">
+
+
+7
+8
+9
+10
+11</pre>
+    </td>
+    <td>
+      <pre class="code"><span class="info file"># File 'lib/quantile_estimator/cursor.rb', line 7</span>
+
+<span class='kw'>def</span> <span class='op'>~</span>
+  <span class='kw'>if</span> <span class='ivar'>@start</span> <span class='op'>&gt;=</span> <span class='int'>0</span>
+    <span class='ivar'>@array</span><span class='lbracket'>[</span><span class='ivar'>@start</span><span class='rbracket'>]</span>
+  <span class='kw'>end</span>
+<span class='kw'>end</span></pre>
+    </td>
+  </tr>
+</table>
+</div>
+    
+  </div>
+
+</div>
+
+    <div id="footer">
+  Generated on Fri Nov 15 15:39:43 2013 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.8.7.3 (ruby-2.0.0).
+</div>
+
+  </body>
+</html>