stream_stats 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34bd7a101e01a96006a3df7971532d0b7b55166d
-  data.tar.gz: c8f925a680d994cdaf03d1aec8768d69bbf93de4
+  metadata.gz: 84b135a85d843c06875ca540fd2a492d932f1a7c
+  data.tar.gz: 2c3747c578d13fa06f344cfa2a2079faa03bdc4f
 SHA512:
-  metadata.gz: 25aebe5f66b5f287ea8ddc150c34ae8a7156c74d338f9bcac2fc25892da6b0d610a9df111c74eedcf03805e1f554323bab7017ef1d29f8f7c29730dc9fb57fe6
-  data.tar.gz: 4ab1046ef06e40bd4cac0ca4b2c75cd99e192aa212a28042a5eeaaf8c21dcc32a917be3b12052918de86d461b1a7b00fe802af4423d54e14ed7f24fe81c0939a
+  metadata.gz: 3ca93d154216b3e4eebe987a214517d8cf9516c14513fed62102a7d0a91104336ae7dc2d20c96a01e606a8aded8be1314d044252f03f378f615e2b0c6d931c6d
+  data.tar.gz: 39870beb67f6d297a944b4c6215f211f8f00a6f2f5a62edfbf794e49ad0555d8dd978f949325a5bace3218519fa51d3a1159c9fbd3c2a39d36993d9e52469082

data/Gemfile

@@ -0,0 +1,11 @@
+source "http://rubygems.org"
+
+gemspec
+
+group :development do
+  gem 'rake'
+end
+
+group :test do
+  gem 'simplecov', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    stream_stats (0.0.4)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    multi_json (1.7.6)
+    rake (10.0.4)
+    rdoc (4.0.0)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  rdoc
+  simplecov
+  stream_stats!

data/README.rdoc

@@ -16,14 +16,16 @@ Create stream and add values
   stream = StreamStats::Stream.new(0.001, [0.50, 0.90])
 
 Parameters:
-  precision level
-  array of quantiles for guaranteed precision
+  1) precision level   
+  2) array of quantiles for guaranteed precision   
 
-The above example guarantees that the 50% and 90% percentile results are guaranteed accurage to +/- 0.001.
+The above example guarantees that the 50% and 90% percentile results are accurate to +/- 0.001.
 
 Populate stream with samples:
 
-  (0..20).each do |i| stream << i end
+  (0..20).each do |i| 
+    stream << i 
+  end
 
 Get stream result whenever desired:
 
@@ -39,5 +41,6 @@ Get stream result whenever desired:
 
 === Credit
 
-Complete credit goes to Armon Dadgar. Algorithm code copied directly out of [statsite](https://github.com/armon/statsite)
+Complete credit goes to Armon Dadgar.  
+Algorithm code copied directly out of [statsite](http://github.com/armon/statsite)
 

data/Rakefile

@@ -0,0 +1,53 @@
+require 'bundler/setup'
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+require 'rdoc/task'
+
+task default: :test
+
+NAME = 'stream_stats'
+
+# rule to build the extension: this says
+# that the extension should be rebuilt
+# after any change to the files in ext
+file "lib/#{NAME}/#{NAME}.#{RbConfig::CONFIG['DLEXT']}" =>
+         Dir.glob("ext/#{NAME}/*{.rb,.c}") do
+  Dir.chdir("ext/#{NAME}") do
+    # this does essentially the same thing
+    # as what RubyGems does
+    ruby "extconf.rb"
+    sh "make"
+  end
+  cp "ext/#{NAME}/#{NAME}.#{RbConfig::CONFIG['DLEXT']}", "lib/#{NAME}"
+end
+
+
+# make the :test task depend on the shared
+# object, so it will be built automatically
+# before running the tests
+task :test => "lib/#{NAME}/#{NAME}.#{RbConfig::CONFIG['DLEXT']}"
+
+# use 'rake clean' and 'rake clobber' to
+# easily delete generated files
+CLEAN.include("ext/**/*{.o,.log,.#{RbConfig::CONFIG['DLEXT']}}")
+CLEAN.include('ext/**/Makefile')
+CLEAN.include("lib/#{NAME}/#{NAME}.#{RbConfig::CONFIG['DLEXT']}")
+CLOBBER.include("lib/**/*.#{RbConfig::CONFIG['DLEXT']}")
+
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation.'
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'StreamStats'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/ext/stream_stats/counter.c

@@ -0,0 +1,103 @@
+#include <math.h>
+#include "counter.h"
+
+/**
+ * Initializes the counter struct
+ * @arg counter The counter struct to initialize
+ * @return 0 on success.
+ */
+int init_counter(counter *counter) {
+    counter->count = 0;
+    counter->sum = 0;
+    counter->squared_sum = 0;
+    counter->min = 0;
+    counter->max = 0;
+    return 0;
+}
+
+/**
+ * Adds a new sample to the struct
+ * @arg counter The counter to add to
+ * @arg sample The new sample value
+ * @return 0 on success.
+ */
+int counter_add_sample(counter *counter, double sample) {
+    if (counter->count == 0) {
+        counter->min = counter->max = sample;
+    } else {
+        if (counter->min > sample)
+            counter->min = sample;
+        else if (counter->max < sample)
+            counter->max = sample;
+    }
+    counter->count++;
+    counter->sum += sample;
+    counter->squared_sum += pow(sample, 2);
+    return 0;
+}
+
+/**
+ * Returns the number of samples in the counter
+ * @arg counter The counter to query
+ * @return The number of samples
+ */
+uint64_t counter_count(counter *counter) {
+    return counter->count;
+}
+
+/**
+ * Returns the mean counter value
+ * @arg counter The counter to query
+ * @return The mean value
+ */
+double counter_mean(counter *counter) {
+    return (counter->count) ? counter->sum / counter->count : 0;
+}
+
+/**
+ * Returns the sample standard deviation counter value
+ * @arg counter The counter to query
+ * @return The sample standard deviation
+ */
+double counter_stddev(counter *counter) {
+    double num = (counter->count * counter->squared_sum) - pow(counter->sum, 2);
+    double div = counter->count * (counter->count - 1);
+    if (div == 0) return 0;
+    return sqrt(num / div);
+}
+
+/**
+ * Returns the sum of the counter
+ * @arg counter The counter to query
+ * @return The sum of values
+ */
+double counter_sum(counter *counter) {
+    return counter->sum;
+}
+
+/**
+ * Returns the sum squared of the counter
+ * @arg counter The counter to query
+ * @return The sum squared of values
+ */
+double counter_squared_sum(counter *counter) {
+    return counter->squared_sum;
+}
+
+/**
+ * Returns the minimum value of the counter
+ * @arg counter The counter to query
+ * @return The minimum value.
+ */
+double counter_min(counter *counter) {
+    return counter->min;
+}
+
+/**
+ * Returns the maximum value of the counter
+ * @arg counter The counter to query
+ * @return The maximum value.
+ */
+double counter_max(counter *counter) {
+    return counter->max;
+}

data/ext/stream_stats/counter.h

@@ -0,0 +1,77 @@
+#ifndef COUNTER_H
+#define COUNTER_H
+#include <stdint.h>
+
+typedef struct {
+    uint64_t count;     // Count of items
+    double sum;         // Sum of the values
+    double squared_sum; // Sum of the squared values
+    double min;         // Minimum value
+    double max;         // Maximum value
+} counter;
+
+/**
+ * Initializes the counter struct
+ * @arg counter The counter struct to initialize
+ * @return 0 on success.
+ */
+int init_counter(counter *counter);
+
+/**
+ * Adds a new sample to the struct
+ * @arg counter The counter to add to
+ * @arg sample The new sample value
+ * @return 0 on success.
+ */
+int counter_add_sample(counter *counter, double sample);
+
+/**
+ * Returns the number of samples in the counter
+ * @arg counter The counter to query
+ * @return The number of samples
+ */
+uint64_t counter_count(counter *counter);
+
+/**
+ * Returns the mean counter value
+ * @arg counter The counter to query
+ * @return The mean value
+ */
+double counter_mean(counter *counter);
+
+/**
+ * Returns the sample standard deviation counter value
+ * @arg counter The counter to query
+ * @return The sample standard deviation
+ */
+double counter_stddev(counter *counter);
+
+/**
+ * Returns the sum of the counter
+ * @arg counter The counter to query
+ * @return The sum of values
+ */
+double counter_sum(counter *counter);
+
+/**
+ * Returns the sum squared of the counter
+ * @arg counter The counter to query
+ * @return The sum squared of values
+ */
+double counter_squared_sum(counter *counter);
+
+/**
+ * Returns the minimum value of the counter
+ * @arg counter The counter to query
+ * @return The minimum value.
+ */
+double counter_min(counter *counter);
+
+/**
+ * Returns the maximum value of the counter
+ * @arg counter The counter to query
+ * @return The maximum value.
+ */
+double counter_max(counter *counter);
+
+#endif

data/ext/stream_stats/stream_stats.c

@@ -7,6 +7,7 @@ VALUE timer_class;
 
 static void strstat_timer_free(void *ptr) {
   destroy_timer(ptr);
+  free(ptr);
 }
 
 static VALUE strstat_timer_init(VALUE self, VALUE rb_eps, VALUE rb_quantiles) {
@@ -37,19 +38,24 @@ static VALUE strstat_timer_init(VALUE self, VALUE rb_eps, VALUE rb_quantiles) {
   init_timer(eps, quantiles, num_quantiles, i_timer);
 
   VALUE data = Data_Wrap_Struct(timer_class, NULL, strstat_timer_free, i_timer);
-  rb_ivar_set(self, rb_intern("timer"), data);
+  rb_ivar_set(self, rb_intern("internal_struct"), data);
 
   return Qnil;
 }
 
+static void *strstat_get_struct(VALUE self) {
+  void *ptr;
+
+  VALUE data = rb_ivar_get(self, rb_intern("internal_struct"));
+  Data_Get_Struct(data, timer, ptr);
+  return ptr;
+}
+
 static VALUE strstat_timer_add_sample(VALUE self, VALUE rb_sample) {
 
   double sample = NUM2DBL(rb_sample);
 
-  timer *i_timer;
-
-  VALUE data = rb_ivar_get(self, rb_intern("timer"));
-  Data_Get_Struct(data, timer, i_timer);
+  timer *i_timer = (timer*) strstat_get_struct(self);
 
   int returned = timer_add_sample(i_timer, sample);
   if (returned != 0) {
@@ -60,10 +66,7 @@ static VALUE strstat_timer_add_sample(VALUE self, VALUE rb_sample) {
 }
 
 static VALUE strstat_timer_count(VALUE self) {
-  timer *i_timer;
-
-  VALUE data = rb_ivar_get(self, rb_intern("timer"));
-  Data_Get_Struct(data, timer, i_timer);
+  timer *i_timer = (timer*) strstat_get_struct(self);
 
   return LONG2NUM(timer_count(i_timer));
 }
@@ -73,10 +76,7 @@ static VALUE strstat_timer_query(VALUE self, VALUE rb_query) {
   if (query < 0 || query > 1)
     rb_raise(rb_eRuntimeError, "invalid quantile");
 
-  timer *i_timer;
-
-  VALUE data = rb_ivar_get(self, rb_intern("timer"));
-  Data_Get_Struct(data, timer, i_timer);
+  timer *i_timer = (timer*) strstat_get_struct(self);
   return DBL2NUM(timer_query(i_timer, query));
 }
 
@@ -89,10 +89,7 @@ static VALUE strstat_timer_percentile(VALUE self, VALUE rb_percentile) {
 }
 
 static VALUE strstat_timer_commoncall(VALUE self, double(*func)(timer*)) {
-  timer *i_timer;
-
-  VALUE data = rb_ivar_get(self, rb_intern("timer"));
-  Data_Get_Struct(data, timer, i_timer);
+  timer *i_timer = (timer*) strstat_get_struct(self);
   return DBL2NUM((*func)(i_timer));
 }
 
@@ -115,6 +112,8 @@ static VALUE strstat_timer_squared_sum(VALUE self) {
   return strstat_timer_commoncall(self, timer_squared_sum);
 }
 
+extern void Init_stream_stats_counter(void);
+
 void Init_stream_stats(void) {
   VALUE module = rb_define_module("StreamStats");
 
@@ -132,4 +131,5 @@ void Init_stream_stats(void) {
   rb_define_method(timer_class, "sum", strstat_timer_sum, 0);
   rb_define_method(timer_class, "squared_sum", strstat_timer_squared_sum, 0);
 
+  Init_stream_stats_counter();
 }

data/ext/stream_stats/stream_stats_counter.c

@@ -0,0 +1,87 @@
+#include <ruby.h>
+#include <stdio.h>
+
+#include "counter.h"
+
+VALUE counter_class;
+
+static VALUE strstat_counter_init(VALUE self) {
+
+  counter *i_counter = (counter *) malloc(sizeof(counter));
+
+  init_counter(i_counter);
+
+  VALUE data = Data_Wrap_Struct(counter_class, NULL, free, i_counter);
+  rb_ivar_set(self, rb_intern("internal_struct"), data);
+
+  return Qnil;
+}
+
+static void *strstat_get_struct(VALUE self) {
+  void *ptr;
+
+  VALUE data = rb_ivar_get(self, rb_intern("internal_struct"));
+  Data_Get_Struct(data, counter, ptr);
+  return ptr;
+}
+
+static VALUE strstat_counter_add_sample(VALUE self, VALUE rb_sample) {
+
+  double sample = NUM2DBL(rb_sample);
+
+  counter *i_counter = (counter*) strstat_get_struct(self);
+
+  int returned = counter_add_sample(i_counter, sample);
+  if (returned != 0) {
+    rb_raise(rb_eRuntimeError, "add sample returned %d", returned);
+  }
+
+  return Qnil;
+}
+
+static VALUE strstat_counter_count(VALUE self) {
+  counter *i_counter = (counter*) strstat_get_struct(self);
+
+  return LONG2NUM(counter_count(i_counter));
+}
+
+static VALUE strstat_counter_commoncall(VALUE self, double(*func)(counter*)) {
+  counter *i_counter = (counter*) strstat_get_struct(self);
+  return DBL2NUM((*func)(i_counter));
+}
+
+static VALUE strstat_counter_min(VALUE self) {
+  return strstat_counter_commoncall(self, counter_min);
+}
+static VALUE strstat_counter_max(VALUE self) {
+  return strstat_counter_commoncall(self, counter_max);
+}
+static VALUE strstat_counter_mean(VALUE self) {
+  return strstat_counter_commoncall(self, counter_mean);
+}
+static VALUE strstat_counter_stddev(VALUE self) {
+  return strstat_counter_commoncall(self, counter_stddev);
+}
+static VALUE strstat_counter_sum(VALUE self) {
+  return strstat_counter_commoncall(self, counter_sum);
+}
+static VALUE strstat_counter_squared_sum(VALUE self) {
+  return strstat_counter_commoncall(self, counter_squared_sum);
+}
+
+void Init_stream_stats_counter(void) {
+  VALUE module = rb_define_module("StreamStats");
+
+  counter_class = rb_define_class_under(module, "Counter", rb_cObject);
+
+  rb_define_method(counter_class, "initialize", strstat_counter_init, 0);
+  rb_define_method(counter_class, "<<", strstat_counter_add_sample, 1);
+  rb_define_method(counter_class, "count", strstat_counter_count, 0);
+  rb_define_method(counter_class, "min", strstat_counter_min, 0);
+  rb_define_method(counter_class, "max", strstat_counter_max, 0);
+  rb_define_method(counter_class, "mean", strstat_counter_mean, 0);
+  rb_define_method(counter_class, "stddev", strstat_counter_stddev, 0);
+  rb_define_method(counter_class, "sum", strstat_counter_sum, 0);
+  rb_define_method(counter_class, "squared_sum", strstat_counter_squared_sum, 0);
+
+}

data/lib/stream_stats/counter.rb

@@ -0,0 +1,11 @@
+module StreamStats
+  class Counter
+
+    def inspect
+      attr_list = [:count, :sum, :min, :max, :mean, :stddev].map do |method|
+        "#{method.to_s}: #{self.send(method)}"
+      end * ', '
+      "#{self.to_s} {#{attr_list}}"
+    end
+  end
+end

data/lib/stream_stats/version.rb

@@ -1,4 +1,3 @@
-class StreamStats
-  VERSION = "0.0.3"
+module StreamStats
+  VERSION = "0.0.4"
 end
-  

data/lib/stream_stats.rb

@@ -1,6 +1,7 @@
-require 'oozie-client/version'
+require 'stream_stats/version'
 require 'stream_stats/stream_stats'
 require 'stream_stats/stream'
+require 'stream_stats/counter'
 
 module StreamStats
 end

data/test/stream_stats/counter_test.rb

@@ -0,0 +1,24 @@
+require 'test_helper'
+
+module StreamStats
+  class CounterTest < MiniTest::Spec
+
+    def test_basic
+      counter = StreamStats::Counter.new
+
+      (0..20).each do |i|
+        counter << i
+      end
+
+      assert_equal 21, counter.count
+      assert_equal 210.0, counter.sum
+      assert_equal 0.0, counter.min
+      assert_equal 20.0, counter.max
+      assert_equal 10.0, counter.mean
+      assert_in_delta 6.2048368229, counter.stddev, 0.0000000001
+
+      assert counter.inspect =~ /#<StreamStats::Counter:0x.*> {count: 21, sum: 210.0, min: 0.0, max: 20.0, mean: 10.0, stddev: 6.2048368229954285}/
+    end
+
+  end
+end

data/test/stream_stats/stream_test.rb

@@ -0,0 +1,29 @@
+require 'test_helper'
+
+module StreamStats
+  class StreamTest < MiniTest::Spec
+
+    def test_basic
+      stream = StreamStats::Stream.new(0.001, [0.50, 0.90])
+
+      (0..20).each do |i|
+        stream << i
+      end
+
+      assert_equal 21, stream.count
+      assert_equal 210.0, stream.sum
+      assert_equal 0.0, stream.min
+      assert_equal 20.0, stream.max
+      assert_equal 10.0, stream.mean
+      assert_in_delta 6.2048368229, stream.stddev, 0.0000000001
+      assert_equal 10.0, stream.quantile(0.50)
+      assert_equal 10.0, stream.percentile(50)
+      assert_equal 18.0, stream.quantile(0.90)
+      assert_equal 18.0, stream.percentile(90)
+      assert_equal Hash[0.50 => 10.0, 0.9 => 18.0], stream.get_quantiles
+
+      assert stream.inspect =~ /#<StreamStats::Stream:0x.*> {count: 21, sum: 210.0, min: 0.0, max: 20.0, mean: 10.0, stddev: 6.2048368229954285, quantiles: {0.5=>10.0, 0.9=>18.0}}/
+    end
+
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+if ENV['COVERAGE']
+  require 'simplecov'
+  SimpleCov.start
+end
+
+require 'rubygems'
+require 'bundler/setup'
+require 'minitest/autorun'
+Bundler.require :default

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stream_stats
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Neville Kadwa
@@ -9,7 +9,21 @@ autorequire:
 bindir: bin
 cert_chain: []
 date: 2013-06-13 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Compute statistics on stream data
 email:
 - neville@kadwa.com
@@ -18,19 +32,28 @@ extensions:
 - ext/stream_stats/extconf.rb
 extra_rdoc_files: []
 files:
-- README.rdoc
+- lib/stream_stats.rb
+- lib/stream_stats/counter.rb
+- lib/stream_stats/stream.rb
+- lib/stream_stats/version.rb
 - ext/stream_stats/cm_quantile.c
 - ext/stream_stats/cm_quantile.h
+- ext/stream_stats/counter.c
+- ext/stream_stats/counter.h
 - ext/stream_stats/extconf.rb
 - ext/stream_stats/heap.c
 - ext/stream_stats/heap.h
 - ext/stream_stats/stream_stats.c
+- ext/stream_stats/stream_stats_counter.c
 - ext/stream_stats/timer.c
 - ext/stream_stats/timer.h
-- lib/stream_stats.rb
-- lib/stream_stats/stream.rb
-- lib/stream_stats/version.rb
-- stream_stats.gemspec
+- Gemfile
+- Gemfile.lock
+- README.rdoc
+- Rakefile
+- test/stream_stats/counter_test.rb
+- test/stream_stats/stream_test.rb
+- test/test_helper.rb
 homepage: http://github.com/kadwanev/stream_stats
 licenses: []
 metadata: {}
@@ -54,4 +77,7 @@ rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Stream Statistics
-test_files: []
+test_files:
+- test/stream_stats/counter_test.rb
+- test/stream_stats/stream_test.rb
+- test/test_helper.rb

data/stream_stats.gemspec

@@ -1,15 +0,0 @@
-# -*- encoding: utf-8 -*-
-require File.expand_path('../lib/stream_stats/version', __FILE__)
-
-Gem::Specification.new do |s|
-  s.name        = 'stream_stats'
-  s.version     = '0.0.1'
-  s.summary     = "Stream Statistics"
-  s.description = "Compute statistics on stream data"
-  s.version     = StreamStats::VERSION
-  s.authors     = ["Neville Kadwa"]
-  s.email       = ["neville@kadwa.com"]
-  s.files       = `git ls-files`.split($\)
-  s.homepage    = 'http://github.com/kadwanev/stream_stats'
-  s.extensions  = ['ext/stream_stats/extconf.rb']
-end