hetchy 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 051d2f9830fb1d6e3bfa363610249fc760ab0ee2
+  data.tar.gz: f2971f802dbc95eea3a1641d5060db28c78cadc7
+SHA512:
+  metadata.gz: 0b9571297d6389a176aae76e58d0b66d55413646c7686cd3f2a85155f8991e6d29c6965649eb26b9b3ecc9eeb447ac15c43f65925184e6fcbd8b415e86922972
+  data.tar.gz: 1f33d20c689b47f2ae6f0db8597f0ac95efe60c83f6feabd1bb6ef3c3c77c4d274e1a20c9bf80ef379ab20920dc13a88f78db5d21dafd0bf9c34f8798cd4f267

data/.gitignore

@@ -0,0 +1,15 @@
+/.ruby-version
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hetchy.gemspec
+gemspec
+
+gem 'minitest', '>= 5'
+gem 'rake'
+gem 'pry'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Matt Sanders
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,113 @@
+# Hetchy
+
+A high performance, thread-safe reservoir sampler for ruby with snapshot and percentile support.
+
+## Benefits/Goals
+
+* Generate statistics for large datasets with a very small in-memory footprint
+* Generate percentiles/quantiles for a known set or numbers or for real-time streaming set
+* Ability to capture sample state at a moment in time for further analysis
+* Speed
+* Thorough test suite
+* No dependencies
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'hetchy'
+```
+
+And then `$ bundle`
+
+Or install it yourself with:
+
+    $ gem install hetchy
+
+## Usage
+
+
+Create a reservoir and designate how big you want it to be:
+
+```ruby
+reservoir = Hetchy::Reservoir.new(size: 1000)
+```
+
+Add samples as they arrive or are generated:
+
+```ruby
+reservoir << 123
+```
+
+You can add sets of samples as well:
+
+```ruby
+reservoir << [45,89,124,96]
+```
+
+You can calculate a percentile at any time:
+
+```ruby
+reservoir.percentile(95)
+```
+
+Hetchy supports high resolution percentiles as well:
+
+```ruby
+reservoir.percentile(99.99)
+```
+
+NOTE: you may need to increase reservoir size for _very_ high resolution percentiles. Experiment to see what works for your data set.
+
+For threaded applications where the reservoir is accepting samples rapidly you can increase performance by snapshotting before running a series of calculations on the reservoir:
+
+```ruby
+dataset  = reservoir.snapshot
+
+perc_95  = dataset.percentile(95)
+perc_99  = dataset.percentile(99)
+perc_999 = dataset.percentile(99.9)
+```
+
+Clear the reservoir to reset it:
+
+```ruby
+reservoir.clear
+```
+
+## Datasets
+
+If you have an existing series you can use Dataset to generate stats for it:
+
+```ruby
+my_series = Array(1..1000)
+dataset = Hetchy::Dataset.new(my_series)
+
+perc_95 = dataset.percentile(95)   #=> 950.95
+median  = dataset.median           #=> 500.5
+```
+
+## Stats Details
+
+For those interested:
+
+* Reservoir sampling is based on Vitter's algorithm R, ensures a uniform sampling probability for every entry in the series
+* Percentile calculations use weighted averages, not nearest neighbor
+
+## Contributing
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project and submit a pull request from a feature or bugfix branch.
+* Please include tests. This is important so we don't break your changes unintentionally in a future version.
+* Please don't modify the gemspec, Rakefile, version, or changelog. If you do change these files, please isolate a separate commit so we can cherry-pick around it.
+
+## Credits
+
+Parts of Hetchy are inspired by Eric Lindvall's [metriks]() gem.
+
+## Copyright
+
+Copyright (c) 2015 Matt Sanders. See LICENSE for details.
+

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/gem_tasks"
+
+# Console
+desc "Open an console session preloaded with this library"
+task :console do
+  sh "pry -r ./lib/hetchy.rb"
+end
+
+# Testing
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task :default => :test

data/hetchy.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hetchy/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hetchy"
+  spec.version       = Hetchy::VERSION
+  spec.authors       = ["Matt Sanders"]
+  spec.email         = ["matt@modal.org"]
+  spec.summary       = %q{High performance, thread-safe reservoir sampler with percentile support.}
+  spec.description   = %q{A high performance, thread-safe reservoir sampler with snapshot and percentile support.}
+  spec.homepage      = "https://github.com/nextmat/hetchy"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  # spec.add_development_dependency "bundler", "~> 1.7"
+  # spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/hetchy.rb

@@ -0,0 +1,5 @@
+module Hetchy; end
+
+require_relative 'hetchy/dataset'
+require_relative 'hetchy/reservoir'
+require_relative 'hetchy/version'

data/lib/hetchy/dataset.rb

@@ -0,0 +1,54 @@
+module Hetchy
+
+  # Takes a fixed immutable dataset (array of values) and provides
+  # various statistical measurements.
+  #
+  class Dataset
+
+    attr_reader :data,
+                :size
+
+    def initialize(data)
+      @data = data.sort
+      @size = @data.length
+    end
+
+    # @return [Float/Integer] median of the data set.
+    def median
+      percentile(50.0)
+    end
+
+    # Generate a percentile for the data set.
+    # @example
+    #  snapshot.percentile(95)
+    #  snapshot.percentile(99.9)
+    #
+    def percentile(perc)
+      if perc > 100.0 || perc < 0.0
+        raise InvalidPercentile, "percentile must be between 0.0 and 100.0"
+      end
+      return 0.0 if data.empty?
+
+      rank = (perc / 100.0) * (size + 1)
+
+      return data[0]          if rank < 1
+      return data[-1]         if rank > size
+      return data[rank - 1]   if rank == Integer(rank)
+      weighted_average_for(rank)
+    end
+
+    private
+
+    # when rank lands between values, generated a weighted average
+    # of adjacent values
+    def weighted_average_for(rank)
+      above = data[rank.to_i]
+      below = data[rank.to_i - 1]
+      fractional = rank - rank.floor
+      below + ((above - below) * fractional)
+    end
+
+  end
+
+  class InvalidPercentile < StandardError; end
+end

data/lib/hetchy/reservoir.rb

@@ -0,0 +1,72 @@
+module Hetchy
+  class Reservoir
+
+    attr_reader :count,     # number of samples processed
+                :pool,      # raw pool data, do not modify this
+                :size       # size of allocated pool
+
+    # Create a reservoir.
+    # @option opts [Integer] :size Size of reservoir
+    #
+    def initialize(opts={})
+      @size = opts.fetch(:size, 1000)
+      @lock = Mutex.new
+      initialize_pool
+    end
+
+    # Add one or more values to the reservoir.
+    # @example
+    #   reservoir << 1234
+    #   reservoir << [2345,7891,2131]
+    #
+    def << (values)
+      Array(values).each do |value|
+        @lock.synchronize do
+          # sampling strategy is Vitter's algo R
+          if count < size
+            @pool[count] = value
+          else
+            index = rand(count+1)
+            if index < @size
+              @pool[index] = value
+            end
+          end
+          @count += 1
+        end
+      end
+    end
+
+    # Empty/reset the reservoir
+    def clear
+      initialize_pool
+    end
+
+    # Calculate a percentile based on the current state of the
+    # reservoir.
+    #
+    # If you are going to calculate multiple percentiles
+    # it will be faster to #snapshot and then calculate them off of
+    # the generated Dataset.
+    def percentile(perc)
+      snapshot.percentile(perc)
+    end
+
+    # Capture a moment in time for the reservoir for analysis.
+    # Since sampling may be ongoing this ensures we are working
+    # with data from our intended period.
+    #
+    def snapshot
+      data = nil
+      @lock.synchronize { data = @pool.dup }
+      Dataset.new(data.compact)
+    end
+
+    private
+
+    def initialize_pool
+      @lock.synchronize { @pool = Array.new(@size) }
+      @count = 0
+    end
+
+  end
+end

data/lib/hetchy/version.rb

@@ -0,0 +1,3 @@
+module Hetchy
+  VERSION = "1.0.0"
+end

data/test/support/sample_sets.rb

@@ -0,0 +1,15 @@
+class SampleSets
+
+  # 100 random samples from 50-2550
+  def self.set_100
+    [1136, 1551, 929, 932, 2383, 970, 2437, 1909, 1083, 2188, 1250, 1530, 1559,
+     782, 2163, 2108, 1173, 2323, 553, 1503, 436, 2418, 2542, 1938, 1276, 328,
+     222, 2384, 2186, 861, 1596, 590, 743, 1837, 1732, 1408, 2177, 2334, 310,
+     1726, 274, 2283, 1908, 138, 587, 325, 975, 2047, 87, 1671, 535, 1007,
+     2340, 1816, 1458, 1565, 1996, 1740, 1568, 853, 1816, 2078, 2094, 707,
+     1444, 939, 2365, 1704, 999, 752, 899, 1220, 2190, 99, 1917, 1738, 2355,
+     1164, 504, 832, 2087, 2311, 2513, 1322, 254, 2486, 1865, 1294, 585, 528,
+     1244, 1395, 1185, 2327, 2461, 2145, 1795, 1336, 1935, 766]
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+require 'bundler'
+Bundler.setup
+
+require 'pry'
+require 'minitest/autorun'
+
+require 'hetchy'
+
+Dir[File.dirname(__FILE__) + '/support/*.rb'].each {|file| require file }

data/test/unit/dataset_test.rb

@@ -0,0 +1,54 @@
+require 'test_helper'
+
+module Hetchy
+  class DatasetTest < Minitest::Test
+
+    def test_data_sort
+      dataset = Dataset.new([5,2,3,1])
+      assert_equal [1,2,3,5], dataset.data, 'data is sorted'
+    end
+
+    def test_median
+      dataset = Dataset.new(Array(1..10))
+      assert_equal 5.5, dataset.median, 'median is accurate'
+    end
+
+    def test_percentile_empty
+      empty_dataset = Dataset.new([])
+      assert_equal 0.0, empty_dataset.percentile(99)
+    end
+
+    def test_percentile_extreme_bounds
+      dataset = Dataset.new(Array(1..9))
+      assert_equal 9, dataset.percentile(99.9)
+      assert_equal 1, dataset.percentile(0.9)
+    end
+
+    def test_percentile_fractional
+      dataset = Dataset.new(SampleSets.set_100)
+      assert_equal 2447.92, dataset.percentile(95.5)
+    end
+
+    def test_percentile_limits
+      dataset = Dataset.new([1,2,3])
+      assert_raises(InvalidPercentile) { dataset.percentile(-0.2) }
+      assert_raises(InvalidPercentile) { dataset.percentile(1000) }
+    end
+
+    def test_percentile_simple
+      dataset = Dataset.new(Array(1..9))
+      assert_equal 5, dataset.percentile(50)
+
+      set = [4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 10]
+      dataset = Dataset.new(set)
+      assert_equal 5, dataset.percentile(25)
+      assert_in_delta 9.85, dataset.percentile(85), 0.001
+    end
+
+    def test_size
+      dataset = Dataset.new([1,2,3])
+      assert_equal 3, dataset.size
+    end
+
+  end
+end

data/test/unit/reservoir_test.rb

@@ -0,0 +1,97 @@
+require 'test_helper'
+
+module Hetchy
+  class ReservoirTest < Minitest::Test
+
+    def test_add_basics
+      # single add
+      reservoir = Reservoir.new(size: 10)
+      reservoir << 1
+      assert_equal 1, reservoir.pool[0]
+
+      # multi add
+      reservoir << [2,3,4]
+      assert_equal 2, reservoir.pool[1]
+      assert_equal 3, reservoir.pool[2]
+      assert_equal 4, reservoir.pool[3]
+    end
+
+    def test_add_pool_fully_accessible
+      series = Array(1..1000)
+      reservoir = Reservoir.new(size: 10)
+      reservoir << series
+
+      refute_equal 1, reservoir.pool[0], 'first entry in pool can be changed'
+      refute_equal 10, reservoir.pool[-1], 'last entry in pool can be changed'
+    end
+
+    def test_add_saturation_starts_sampling
+      series = Array(1..15)
+      overflow = series[10..-1]
+
+      reservoir = Reservoir.new(size: 10)
+      reservoir << series
+
+      assert_equal 15, reservoir.count, 'all samples recorded'
+      assert_equal 10, reservoir.pool.length, 'pool is stable size'
+      refute_equal (series - reservoir.pool), overflow,
+        'overflow measures are sampled into pool'
+    end
+
+    def test_add_thread_safety
+      reservoir = Reservoir.new(size: 1024)
+      threads = []
+      5.times do
+        threads << Thread.new do
+          1000.times { reservoir << rand(10000) }
+        end
+      end
+
+      threads.each(&:join)
+      assert_equal 5000, reservoir.count, 'adding is threadsafe'
+    end
+
+    def test_clear
+      reservoir = Reservoir.new(size: 10)
+      reservoir << [1,2,3]
+
+      reservoir.clear
+      assert_nil reservoir.pool[0], 'should clear reservoir'
+      assert_equal 0, reservoir.count
+    end
+
+    def test_count
+      reservoir = Reservoir.new(size: 10)
+      assert_equal 0, reservoir.count
+
+      10.times do
+        reservoir << rand(1000)
+      end
+      assert_equal 10, reservoir.count
+    end
+
+    def test_percentile
+      reservoir = Reservoir.new(size: 10)
+      reservoir << [1,2,3,4,5]
+
+      assert_equal 3, reservoir.percentile(50)
+    end
+
+    def test_size
+      reservoir = Reservoir.new(size: 5)
+      assert_equal 5, reservoir.size
+    end
+
+    def test_snapshot
+      reservoir = Reservoir.new(size: 5)
+      reservoir << [1,2]
+      snapshot = reservoir.snapshot
+
+      reservoir << 3
+      assert_includes snapshot.data, 1, 'has previous samples'
+      assert_includes snapshot.data, 2, 'has previous samples'
+      refute_includes snapshot.data, 3, 'does not receive changes'
+    end
+
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: hetchy
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Matt Sanders
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-27 00:00:00.000000000 Z
+dependencies: []
+description: A high performance, thread-safe reservoir sampler with snapshot and percentile
+  support.
+email:
+- matt@modal.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- hetchy.gemspec
+- lib/hetchy.rb
+- lib/hetchy/dataset.rb
+- lib/hetchy/reservoir.rb
+- lib/hetchy/version.rb
+- test/support/sample_sets.rb
+- test/test_helper.rb
+- test/unit/dataset_test.rb
+- test/unit/reservoir_test.rb
+homepage: https://github.com/nextmat/hetchy
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: High performance, thread-safe reservoir sampler with percentile support.
+test_files:
+- test/support/sample_sets.rb
+- test/test_helper.rb
+- test/unit/dataset_test.rb
+- test/unit/reservoir_test.rb