streaming_stats 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8f8c9052c66697c167070b7855fe5e7964ebaed158a698565e8592d510b16562
+  data.tar.gz: 204081b5f172a2cdd03ddeabd3df30ac4d89e89d2b50a0796e81cc6bd19d778a
+SHA512:
+  metadata.gz: 74456f61026259b87b9a5f8a08c263823183e825305cfcc9ce9451571b56337a3a1a5bdab4335b974df3273c4fcf713d32d322973ed88682ce8c3a3857737000
+  data.tar.gz: c9f2921fed3e04a268d79fec6002f85a2393174c822ae8873d70fcfa8a1eaad66f05d252165ab4ff0a6716c1362ad42090aaef7a75f0df1c96249c4a24fd45c0

data/Gemfile

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gem 'activesupport', '~> 5.2'
+
+group :development, :test do
+  gem 'byebug'
+  gem 'distribution'
+  gem 'guard'
+  gem 'guard-minitest'
+  gem 'guard-rubocop'
+  gem 'minitest'
+  gem 'mocha'
+  gem 'pry'
+  gem 'rake'
+  gem 'terminal-notifier-guard'
+  # If you update rubocop or rubocop-rails, please also update Gemfile.linting
+  gem 'rubocop', '~> 0.5'
+  gem 'rubocop-performance'
+end
+
+group :development do
+  gem 'simplecov'
+end

data/Gemfile.lock

@@ -0,0 +1,108 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.4.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    ast (2.4.1)
+    byebug (11.1.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.7)
+    distribution (0.8.0)
+    docile (1.3.2)
+    ffi (1.13.1)
+    formatador (0.2.5)
+    guard (2.16.2)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (>= 1.0.12, < 2.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-minitest (2.4.6)
+      guard-compat (~> 1.2)
+      minitest (>= 3.0)
+    guard-rubocop (1.4.0)
+      guard (~> 2.0)
+      rubocop (< 2.0)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
+    listen (3.3.3)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    lumberjack (1.2.8)
+    method_source (1.0.0)
+    minitest (5.14.2)
+    mocha (1.11.2)
+    nenv (0.3.0)
+    notiffany (0.1.3)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    parallel (1.20.1)
+    parser (2.7.2.0)
+      ast (~> 2.4.1)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    rb-fsevent (0.10.4)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    regexp_parser (2.0.0)
+    rexml (3.2.4)
+    rubocop (0.93.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.5)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 0.6.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.3.0)
+      parser (>= 2.7.1.5)
+    rubocop-performance (1.9.1)
+      rubocop (>= 0.90.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    ruby-progressbar (1.10.1)
+    shellany (0.0.1)
+    simplecov (0.20.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.2)
+    terminal-notifier-guard (1.7.0)
+    thor (1.0.1)
+    thread_safe (0.3.6)
+    tzinfo (1.2.8)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 5.2)
+  byebug
+  distribution
+  guard
+  guard-minitest
+  guard-rubocop
+  minitest
+  mocha
+  pry
+  rake
+  rubocop (~> 0.5)
+  rubocop-performance
+  simplecov
+  terminal-notifier-guard
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Will Fitzgerald
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,58 @@
+# Streaming Stats
+
+![Ruby](https://github.com/willf/streaming-stats/workflows/Ruby/badge.svg)
+
+StreamingStats is a Ruby class that takes streaming numeric data
+and return descriptive statistics with minimal overhead.
+A stream with n entries will only require about log2(n) storage.
+The main update function is `insert`, and the object can
+return:
+
+- n (number of values inserted)
+- sum
+- mean
+- stddev
+- variance
+- quantile (i.e. percentile)
+- min
+- max
+
+Note that quantiles are approximate.
+
+```irb
+require 'streaming_stats'
+> gk = StreamingStats.new(epsilon: 0.01)
+> 10000.times {gk.insert rand}
+=> 10000
+> gk.n
+=> 10000
+> gk.sum
+=> 4985.484627445102
+> gk.mean
+=> 0.4985484627445139
+> gk.stddev
+=> 0.288236161831176
+> gk.variance
+=> 0.08308008498716787
+> gk.min
+=> 0.0001414880872682156
+> gk.max
+=> 0.9999396732975679
+> gk.quantile 0.1
+=> 0.08869274826771956
+> gk.quantile 0.5
+=> 0.4944707523857559
+> gk.quantile 0.9
+=> 0.9004683944698589
+> gk.quantile 0.999
+=> 0.9999396732975679
+gk.compression_ratio
+=> 0.9927
+```
+
+The basic stats (n, sum, mean, variance, stddev) are from 
+my very first Gist: https://gist.github.com/willf/187846.
+
+The approximate quartile method is a port of [streaming-percentiles-js](https://github.com/sengelha/streaming-percentiles-js).
+
+ How to calculate streaming percentiles is discussed in Steven Englehardt's series, [Calculating Percentiles on Streaming Data](https://www.stevenengelhardt.com/series/calculating-percentiles-on-streaming-data/).

data/Rakefile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'rake/testtask'
+
+Rake::TestTask.new :test do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/test_*.rb'
+end
+
+desc 'Run tests and linter'
+task :default do
+  sh %(script/lint)
+  sh %(script/test)
+end

data/lib/streaming_stats.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require 'ostruct'
+
+# Public: StreamingStats class
+# StreamingStats is a Ruby class that takes streaming numeric data
+# and return descriptive statistics with minimal overhead.
+# A stream with n entries will only require about log2(n) storage.
+# The main update function is `insert`, and the object can
+# return
+# - n (number of values inserted)
+# - sum
+# - mean
+# - stddev
+# - variance
+# - quantile (i.e. percentile)
+# - min
+# - max
+# The sum, mean, stddev, variance functions are calculated more or less
+# as in the technical description here: https://www.johndcook.com/blog/standard_deviation/
+#
+# The quantile method is a Ruby port of https://github.com/sengelha/streaming-percentiles-js
+# The variable names, etc. of the quantile method are adopted from that project
+#
+# The compression size can be estimated with the method compression_size
+#
+# require 'streaming_stats'
+# > gk = StreamingStats.new(epsilon: 0.01); 10000.times {gk.insert rand}
+# => 10000
+# > gk.n
+# => 10000
+# > gk.sum
+# => 4985.484627445102
+# > gk.mean
+# => 0.4985484627445139
+# > gk.stddev
+# => 0.288236161831176
+# > gk.variance
+# => 0.08308008498716787
+# > gk.min
+# => 0.0001414880872682156
+# > gk.max
+# => 0.9999396732975679
+# > gk.quantile 0.1
+# => 0.08869274826771956
+# > gk.quantile 0.5
+# => 0.4944707523857559
+# > gk.quantile 0.9
+# => 0.9004683944698589
+# > gk.quantile 0.999
+# => 0.9999396732975679
+# gk.compression_ratio
+# => 0.9927
+class StreamingStats
+  GK_MAX_BAND = 999_999
+  attr_reader :epsilon, :n, :mean, :sum
+
+  # epsilon - "epsilon is allowable error. As epsilon becomes smaller,
+  # the accuracy of the approximation improves, but the class
+  # consumes more memory" see https://www.stevenengelhardt.com/series/calculating-percentiles-on-streaming-data/
+  def initialize(epsilon: 0.1)
+    @n = 0
+    @mean = 0.0
+    @m2 = 0.0
+    @sum = 0.0
+
+    @epsilon = epsilon
+    @one_over_2e = 1 / (2 * epsilon)
+    @S = []
+  end
+
+  # Public: Returns the compression list
+  # For debugging only
+  def s
+    @S
+  end
+
+  # Public: inserts a value from a stream, updating the state
+  #
+  # value - The Numeric to be inserted
+  #
+  # Examples
+  #
+  #   insert(100)
+  #   => 100
+  #
+  # Returns the Numeric inserted
+  def insert(value)
+    ## Basic stats accumulators
+    @n += 1
+    @sum += value
+    delta = value - @mean
+    @mean += (delta / @n)
+    @m2 += (delta * (value - @mean))
+    ## quantile work
+    _compress if (@n % @one_over_2e).zero?
+    _do_insert value
+    value
+  end
+
+  # Public: Returns the variance of the streamed data. Initialized to 0.0
+  #
+  # Examples
+  #
+  #   variance
+  #   => 2.00
+  #
+  # Returns the variance
+  def variance
+    return 0 if @n <= 1
+
+    @m2 / @n
+  end
+
+  # Public: Returns the standard deviation of the streamed data. Initialized to 0.0
+  #
+  # Examples
+  #
+  #   stddev
+  #   => 1.414
+  #
+  # Returns the standard deviation
+  def stddev
+    Math.sqrt(variance)
+  end
+
+  # Public: Returns the approximate quantile (percentile) at phi
+  #
+  # phi - A Numeric between 0.0 and 1.0, inclusive
+  #
+  # Examples
+  #
+  #   quantile(0.5)
+  #   => 5.01
+  #
+  # Returns the approximate quantile
+  def quantile(phi)
+    throw ArgumentError.new("#{phi} must be between 0.0 and 1.0 inclusive") unless phi.between?(0.0, 1.0)
+    en = @epsilon * @n
+    r = (phi * @n).ceil
+    rmin = 0
+    (0..@S.size - 1).each do |i|
+      rmin += @S[i].g
+      rmax = rmin + @S[i].delta
+      return @S[i].v if r - rmin <= en && rmax - r <= en
+    end
+    throw 'Unknown error'
+  end
+
+  # Public: Returns the minimum value so far inserted
+  #
+  # Examples
+  #
+  #   max
+  #   => 500.0
+  #
+  # Returns the minimum value
+  def min
+    @S[0].v
+  end
+
+  # Public: Returns the maximum value so far inserted
+  #
+  # Examples
+  #
+  #   max
+  #   => 500.0
+  #
+  # Returns the maximum value
+  def max
+    @S.last.v
+  end
+
+  # Public: Returns the compression ratio achieved
+  #
+  # Examples
+  #
+  #   compression_ration
+  #   => 99.1
+  #
+  # Returns the compression ratio achieved
+  def compression_ratio
+    1.0 - (1.0 * @S.size / @n)
+  end
+
+  # Private: Compresses the number of values stored
+  def _compress
+    two_epsilon_n = 2 * @epsilon * @n
+    bands = StreamingStats._construct_band_lookup(two_epsilon_n)
+    # We must always keep the first and last nodes as these
+    # are global min/max
+    i = @S.length - 2
+    while i >= 1
+      if bands[@S[i].delta] <= bands[@S[i + 1].delta]
+        start_indx = i
+        g_i_star = @S[i].g
+        while start_indx >= 2 && (bands[@S[start_indx - 1].delta] < bands[@S[i].delta])
+          start_indx -= 1
+          g_i_star += @S[start_indx].g
+        end
+        if (g_i_star + @S[i + 1].g + @S[i + 1].delta) < two_epsilon_n
+          # The below is a delete_tuples([start_indx, i]) operation
+          merged = OpenStruct.new(
+            v: @S[i + 1].v,
+            g: g_i_star + @S[i + 1].g,
+            delta: @S[i + 1].delta
+          )
+          StreamingStats.splice!(@S, start_indx, 2 + (i - start_indx), merged)
+          i = start_indx
+        end
+      end
+      i -= 1
+    end
+  end
+
+  # Private: Constructs a band lookup
+  def self._construct_band_lookup(two_epsilon_n)
+    bands = Array.new(two_epsilon_n + 1)
+    bands[0] = GK_MAX_BAND
+    bands[two_epsilon_n] = 0 # when float?
+    p = two_epsilon_n.floor
+    (1..Math.log2(two_epsilon_n).ceil).each do |alpha|
+      two_alpha_minus_1 = 2**(alpha - 1)
+      two_alpha = 2**alpha
+      lower = [p - two_alpha - (p % two_alpha), 0].max
+      upper = p - two_alpha_minus_1 - (p % two_alpha_minus_1)
+      ((lower + 1)..upper).each do |i|
+        bands[i] = alpha
+      end
+    end
+    bands
+  end
+
+  # Private: Actually does a new insertion into S
+  def _do_insert(v)
+    i = _find_insertion_index(v)
+    delta = _determine_delta(i)
+    tuple = OpenStruct.new(v: v, g: 1, delta: delta)
+    StreamingStats.splice!(@S, i, 0, tuple)
+    @S
+  end
+
+  # Private: Find where to insert
+  def _find_insertion_index(value)
+    i = 0
+    i += 1 while i < @S.size && value >= @S[i].v
+    i
+  end
+
+  # Private: Determine delta
+  def _determine_delta(i)
+    return 0 if @n < @one_over_2e
+    return 0 if i.zero? || i == @S.size
+
+    (2 * @epsilon * @n).floor - 1
+  end
+
+  # from https://stackoverflow.com/questions/6892551/array-prototype-splice-in-ruby
+  # Same as Javascript splice, but not put on Array prototype
+  def self.splice!(array, start, len, *replacements)
+    r = array.slice!(start, len)
+    array[start, 0] = replacements if replacements
+    r
+  end
+end

data/script/console

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Usage: script/console
+
+set -e
+
+bundle exec pry -r ./streaming_stats.rb $@

data/script/lint

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -e
+
+bundle exec rubocop . $@ 

data/script/lint-autofix

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -e
+
+bundle exec rubocop . --auto-correct

data/script/test

@@ -0,0 +1,14 @@
+#!/bin/sh
+#/ Usage: test
+#/
+#/ Run the tests.
+
+set -e
+cd $(dirname "$0")/..
+
+[ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && {
+    grep '^#/' <"$0"| cut -c4-
+    exit 0
+}
+
+bundle exec rake test $@

data/streaming_stats.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'bundler/setup'
+require 'active_support/all'
+
+require './lib/streaming_stats'

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require_relative '../streaming_stats'

data/test/test_streaming_stats.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class StreamingStatsTest < Minitest::Test
+  def test_default_initialization
+    gk = StreamingStats.new
+    assert_equal gk.epsilon, 0.1
+  end
+
+  def test_given_init
+    gk = StreamingStats.new(epsilon: 0.2)
+    assert_equal gk.epsilon, 0.2
+  end
+
+  def test_insert
+    gk = StreamingStats.new
+    300.times do |i|
+      assert_equal gk.n, i
+      gk.insert i
+      assert_equal gk.n, i + 1
+    end
+  end
+
+  def test_basic_stats
+    gk = StreamingStats.new(epsilon: 0.01)
+    1_000.times do
+      gk.insert rand
+    end
+    assert_in_delta gk.mean, 0.5, 0.03
+    assert_in_delta gk.variance, 1 / 12.0, 0.05
+    assert_in_delta gk.stddev, Math.sqrt(1 / 12.0), 0.05
+    assert_equal gk.n, 1000
+    assert_in_delta gk.sum, gk.mean * gk.n, 0.01
+  end
+
+  def test_initialized_stats
+    gk = StreamingStats.new(epsilon: 0.01)
+    assert_in_delta gk.mean, 0.0, 0.001
+    assert_in_delta gk.variance, 0.0, 0.001
+    assert_in_delta gk.stddev, 0.0, 0.001
+    assert_equal gk.n, 0
+    assert_in_delta gk.sum, 0.0, 0.001
+  end
+
+  def test_quantiles
+    gk = StreamingStats.new(epsilon: 0.01)
+    10_000.times do
+      gk.insert rand
+    end
+    assert_in_delta gk.quantile(0.1), 0.1, 0.03
+    assert_in_delta gk.quantile(0.5), 0.5, 0.03
+    assert_in_delta gk.quantile(0.5), gk.mean, 0.03
+    assert_in_delta gk.quantile(0.9), 0.9, 0.03
+  end
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: streaming_stats
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Will Fitzgerald
+autorequire:
+bindir: script
+cert_chain: []
+date: 2020-10-08 00:00:00.000000000 Z
+dependencies: []
+description: Calculates descriptive statistics from streams with minimal overhead
+email: will.fitzgerald@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- lib/streaming_stats.rb
+- script/console
+- script/lint
+- script/lint-autofix
+- script/test
+- streaming_stats.rb
+- test/test_helper.rb
+- test/test_streaming_stats.rb
+homepage: https://github.com/willf/streaming_stats
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Calculates descriptive statistics from streams
+test_files: []