running_stat 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8ec11d286e66a056fc873beecd760b3916008ea2
+  data.tar.gz: 99915adeadbbe43c59c190a5392f46c85f438126
+SHA512:
+  metadata.gz: dc655fd6ca351e6bf5a3d9fb4331b2e704b22cd0a3551dd914ae025ad558b2a261dad16edbf7964578fc4945a7d3b54da013fc9040e83c8fb2929483f4e1e416
+  data.tar.gz: 5ed907c0cc481824640b8df911556c509616132e991ab1a9fecf347cb54c22398132990906bff31d6a6fac7cb1d5f7848b92509486dc3b9e620043867cd6178d

data/.gitignore

@@ -0,0 +1,12 @@
+.ruby-version
+.ruby-gemset
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.sw?

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+rvm:
+  - jruby-1.7
+  - jruby-9k
+  - 1.8.7
+  - 1.9.3
+  - 2.2
+before_install: gem install bundler -v 1.10.6
+services:
+  - redis-server

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## [0.0.1] - 2016-01-11
+### Added
+- Initial release
+
+### Changed
+- n/a

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Anuj Das
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,81 @@
+# RunningStat
+
+[![Build Status](https://travis-ci.org/anujdas/running_stat.png?branch=master)](https://travis-ci.org/anujdas/running_stat)
+
+[![Gem Version](https://badge.fury.io/rb/running_stat.png)](http://badge.fury.io/rb/running_stat)
+
+RunningStat provides distributed redis-backed data buckets on which various statistics are calculated live without storing the datapoints themselves: cardinality, average (arithmetic mean), standard deviation, and variance. Numbers (integer or float) can be pushed into buckets atomically. The space and time overhead for each metric is constant and invariant under data cardinality.
+
+The algorithm used is based on Knuth's TAOCP and is numerically stable; a brief writeup is available on Wikipedia under [Algorithms for calculating online variances](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm).
+
+Disclaimer: Because it uses atomic Lua scripting, RunningStat requires redis 2.6+.
+
+
+## Quickstart
+
+RunningStat is easy to use. Since it depends on redis, you need to have a configured [Redis](https://github.com/redis/redis-rb) instance ([ConnectionPool](https://github.com/mperham/connection_pool) wrapper highly recommended/practically mandatory if your app is multithreaded). By default, RunningStat will use `Redis.current`, but you can pass a redis instance to the constructor in the `:redis` key, for example if you're using ConnectionPool:
+
+```ruby
+stat = RunningStat.instance('my_bucket')  # uses Redis.current
+# or
+stat = RunningStat.instance('my_bucket', redis: Redis.new(db: 1))  # uses db 1 for stats
+# or
+stat = $redis_pool.with { |redis| RunningStat.new('my_bucket', redis: redis) }  # checks out from pool
+```
+
+Now, for anything you want to measure, pick a bucket name and push your data:
+
+```ruby
+stat = RunningStat.instance('my_bucket')
+stat.push(1)
+stat.push(100.0)
+stat.push(-10)
+```
+
+At any point, you can obtain stats about the data seen so far in a given bucket:
+
+```ruby
+> stat = RunningStat.instance('my_bucket')
+ => #<RunningStat instance>
+> stat.cardinality
+ => 3
+> stat.mean
+ => 30.333333333333
+> stat.std_dev
+ => 60.58327602014685
+> stat.variance
+ => 3670.3333333333
+```
+
+Reads and writes are both O(1); stats are calculated on insert, so reads are fast.
+
+Note that by definition, none of the statistics except cardinality are defined for datasets of cardinality < 2; you'll see a RunningStat::InsufficientDataError raised instead.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'running_stat'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install running_stat
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'running_stat'
+
+require 'irb'
+IRB.start

data/bin/setup

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install

data/lib/running_stat/insufficient_data_error.rb

@@ -0,0 +1,6 @@
+class RunningStat
+  # Raised when < 2 data points are provided
+  class InsufficientDataError < RuntimeError
+    ERROR_STRING = 'Insufficient Data'.freeze
+  end
+end

data/lib/running_stat/invalid_data_error.rb

@@ -0,0 +1,5 @@
+class RunningStat
+  # Raised when a non-numerical value is pushed to a state
+  class InvalidDataError < ArgumentError
+  end
+end

data/lib/running_stat/lua/push_datum.rb

@@ -0,0 +1,31 @@
+require 'digest/sha1'
+
+class RunningStat
+  module Lua
+    # Keys:
+    #   count - the dataset cardinality
+    #   mean - the mean of the dataset so far
+    #   m2 - the running sum of the squares of the differences of the dataset
+    # Arguments:
+    #   datum - the number to be added to the dataset
+    # Effects:
+    #   calculates running stats (mean, variance, std_dev)
+    # Returns:
+    #   nothing
+    PUSH_DATUM = <<-EOLUA
+      local count_key = KEYS[1]
+      local mean_key = KEYS[2]
+      local m2_key = KEYS[3]
+      local datum = ARGV[1]
+
+      local mean = tonumber(redis.call("GET", mean_key)) or 0.0
+      local delta = datum - mean
+
+      local count = redis.call("INCR", count_key)
+      mean = redis.call("INCRBYFLOAT", mean_key, tostring(delta / count))
+      redis.call("INCRBYFLOAT", m2_key, tostring(delta * (datum - mean)))
+    EOLUA
+
+    PUSH_DATUM_SHA1 = Digest::SHA1.hexdigest(PUSH_DATUM).freeze
+  end
+end

data/lib/running_stat/lua/variance.rb

@@ -0,0 +1,30 @@
+require 'digest/sha1'
+require 'running_stat/insufficient_data_error'
+
+class RunningStat
+  module Lua
+    # Keys:
+    #   count - the dataset cardinality
+    #   m2 - the running sum of the squares of the differences of the dataset
+    # Arguments:
+    #   nothing
+    # Effects:
+    #   nothing
+    # Returns:
+    #   the sample variance of the dataset, as a stringified float
+    VARIANCE = <<-EOLUA
+      local count_key = KEYS[1]
+      local m2_key = KEYS[2]
+
+      local count = tonumber(redis.call("GET", count_key)) or 0
+      if count < 2 then
+        return redis.error_reply("#{InsufficientDataError::ERROR_STRING}")
+      else
+        local m2 = tonumber(redis.call("GET", m2_key)) or 0.0
+        return tostring(m2 / (count - 1))
+      end
+    EOLUA
+
+    VARIANCE_SHA1 = Digest::SHA1.hexdigest(VARIANCE).freeze
+  end
+end

data/lib/running_stat/version.rb

@@ -0,0 +1,3 @@
+class RunningStat
+  VERSION = '0.0.1'
+end

data/lib/running_stat.rb

@@ -0,0 +1,78 @@
+require 'running_stat/version'
+
+require 'redis'
+
+require 'running_stat/lua/push_datum'
+require 'running_stat/lua/variance'
+require 'running_stat/insufficient_data_error'
+require 'running_stat/invalid_data_error'
+
+class RunningStat
+  BASE_KEY = 'running_stat:v1'
+
+  # Returns an instance of RunningStat for the given dataset
+  def self.instance(data_bucket, opts = {})
+    new(data_bucket, opts)
+  end
+
+  def initialize(data_bucket, opts = {})
+    @data_bucket = data_bucket
+    @redis = opts[:redis]
+  end
+
+  # Adds a piece of numerical data to the dataset's stats
+  def push(datum)
+    redis.eval(Lua::PUSH_DATUM, [count_key, mean_key, sum_sq_diff_key], [Float(datum)])
+  rescue ArgumentError => e
+    raise InvalidDataError.new(e)  # datum was non-numerical
+  end
+
+  # Returns the number of data points seen, or 0 if the stat does not exist
+  def cardinality
+    redis.get(count_key).to_i
+  end
+
+  # Returns the arithmetic mean of data points seen, or 0.0 if non-existent
+  def mean
+    redis.get(mean_key).to_f
+  end
+
+  # Returns the sample variance of the dataset so far, or raises
+  # an InsufficientDataError if insufficient data (< 2 datapoints)
+  # has been pushed
+  def variance
+    redis.eval(Lua::VARIANCE, [count_key, sum_sq_diff_key], []).to_f
+  rescue Redis::CommandError => e
+    raise InsufficientDataError.new(e)  # only CommandError possible
+  end
+
+  # Returns the standard deviation of the dataset so far, or raises
+  # an InsufficientDataError if insufficient data (< 2 datapoints)
+  # has been pushed
+  def std_dev
+    Math.sqrt(variance)
+  end
+
+  # Resets the stat to reflect an empty dataset
+  def flush
+    redis.del(count_key, mean_key, sum_sq_diff_key)
+  end
+
+  private
+
+  def redis
+    @redis || Redis.current
+  end
+
+  def count_key
+    "#{BASE_KEY}:#{@data_bucket}:count"
+  end
+
+  def mean_key
+    "#{BASE_KEY}:#{@data_bucket}:mean"
+  end
+
+  def sum_sq_diff_key
+    "#{BASE_KEY}:#{@data_bucket}:sum_sq_diff"
+  end
+end

data/running_stat.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'running_stat/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'running_stat'
+  spec.version       = RunningStat::VERSION
+  spec.authors       = ['Anuj Das']
+  spec.email         = ['anujdas@gmail.com']
+
+  spec.summary       = %q{A distributed streaming mean, variance, and standard deviation metric}
+  spec.description   = %q{Using redis, allows statistics calculations on a streaming set of data without storing every value}
+  spec.homepage      = 'https://www.github.com/anujdas/running_stat'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'redis', '~> 3.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: running_stat
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Anuj Das
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-01-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Using redis, allows statistics calculations on a streaming set of data
+  without storing every value
+email:
+- anujdas@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/running_stat.rb
+- lib/running_stat/insufficient_data_error.rb
+- lib/running_stat/invalid_data_error.rb
+- lib/running_stat/lua/push_datum.rb
+- lib/running_stat/lua/variance.rb
+- lib/running_stat/version.rb
+- running_stat.gemspec
+homepage: https://www.github.com/anujdas/running_stat
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: A distributed streaming mean, variance, and standard deviation metric
+test_files: []