ministat 1.2.6 → 1.3.0

data/History.rdoc

@@ -1,3 +1,10 @@
+== 1.3.0 / 2011-10-16
+
+* Fixing a typo in geometric mean
+* Bug-fix in variance
+* New tests
+* Use Minitest explicitly
+
 == 1.2.4 / 2010-09-18
 
 * Version bump for packaging

data/Manifest.txt

@@ -4,6 +4,8 @@ README.rdoc
 Rakefile
 bin/ministat
 lib/ministat.rb
+lib/ministat/data.rb
 test/data/1.dat
 test/data/2.dat
+test/data/3.dat
 test/test_ministat.rb

data/lib/ministat.rb

@@ -1,156 +1,5 @@
-require 'mathn'
+require 'ministat/data'
 
 module MiniStat
-  VERSION = '1.2.6'
-  class Data
-    attr_reader :data
-
-    def initialize(data)
-      @data = data.collect {|data| data.to_f}.sort
-      @sorted = true
-    end
-
-    def <<(obj)
-      throw "#{obj.to_s} is not numeric" unless obj.to_f
-      @data << obj
-      # force computation!
-      @q1 = @q3 = @iqr = @outliers = @std_dev = @variance = 
-      @mode = @harmonic_mean = @geometric_mean = nil
-    end
-
-    # Return the median of your dataset. Naive implementaion
-    # -- does a sort on the data.
-    def median(data=@data)
-      unless @sorted and data == @data
-        data.sort!
-        @sort = true
-      end
-      if data.size % 2 == 0
-        return (data[data.size / 2.0 - 1] + data[(data.size / 2.0)]) / 2
-      else 
-        # split = (data.size - 1) / 2
-        # this was just weird. i need to look this up
-        # return (data[split - 1.5] + data[split - 0.5]) / 2
-        return data[(data.size - 1)/2]
-      end
-    end
-
-    def partition(pivot, data=@data)
-      low  = []
-      high = []
-      data.each do |i|
-        high.push(i) if i > pivot
-        low.push(i)  if i < pivot
-      end
-      return {:low => low, :high => high}
-    end
-
-    # First quartile.
-    def q1
-      @q1 ||= median(partition(median(@data), @data)[:low])
-    end
-
-    # Third quartile
-    def q3
-      @q3 ||= median(partition(median(@data), @data)[:high])
-    end
-
-    # Interquartile range, ie, the middle 50% of the data.
-    def iqr
-      @iqr ||= q3 - q1
-    end
-
-    # Returns an array of outlying data points.
-    def outliers
-      @outliers ||= 
-        @data.collect do |i|
-          i if (i < q1 - (1.5 * iqr) or i > q3 + (1.5 * iqr))
-        end.compact
-    end
-
-    # Computes arthmetic mean (most common average).
-    def mean(data=@data)
-      @mean = (data.inject(0) {|i,j| i += j}) / data.size
-    end
-
-    # Computes mode and generates a histogram (for free!). 
-    # (We needed it anyway).
-    def mode
-      @hist     ||= {}
-      @max_freq ||= 0
-      @mode     ||= nil
-      unless @mode
-        @data.each do |val|
-          @hist[val] ||= 0
-          @hist[val] += 1
-          @max_freq, @mode = @hist[val], val if @hist[val] > @max_freq
-        end
-      end
-      @mode
-    end
-
-    # Computes variance. Used to measure degree of spread 
-    # in dataset.
-    def variance
-      @variance ||= 
-        @data.inject(0) { |i,j| i += (j - mean(@data)) ** 2}  / (@data.size)
-    end
-
-    # Standard deviation. Square root of variance, measure of the
-    # spread of the data about the mean.
-    def std_dev
-      @std_dev ||= Math.sqrt(variance)
-    end
-
-    # Geometric mean. Only applies to non-negative numbers, and
-    # relates to log-normal distribution.
-    def geometric_mean
-      @geoeteric_mean ||= 
-        (@data.inject(1) {|i,j| i *= j})**(1.0/@data.size)
-    end
-
-    # Harmonic or subcontrary mean. Tends strongly toward the least
-    # elements of the dataset.
-    def harmonic_mean
-      @harmonic_mean ||=
-         @data.size.to_f / (@data.inject(0) {|i,j| i += (1.0/j)})
-    end
-
-    # Put the histogram into a string if we have it
-    def hist
-      if defined? @hist
-        # this is a textbook example of how to lie with statistics...
-        # TODO: iterate over a range rather than @hist.keys--a histogram
-        # produced out of the keys won't properly represent flat spots
-        # with no data. or something like that. do as i say, not as i do.
-        #
-        # this code borrows liberally from the ruby cookbook, recipe 5.12
-        # ORA, 2006
-        pairs = @hist.keys.collect { |x| [x.to_s, @hist[x]] }.sort
-        largest_key_size = pairs.max {|x,y| x[0].size <=> y[0].size }[0].size
-        pairs.inject("") do |s,kv|
-        s<< "#{kv[0].ljust(largest_key_size)} |#{char*kv[1]}\n"
-      end
-      end
-    end
-
-    # Return a string with statisical info about a dataset.
-    def to_s
-      <<-DATA_STR    
-        Partition:#{partition(median).inspect} 
-        Mean:#{mean}
-        Geometric Mean:#{geometric_mean}
-        Harmonic Mean:#{harmonic_mean}
-        Median:#{median} 
-        Min:#{data.min} 
-        Q1:#{q1}
-        Q3:#{q3}
-        Max:#{data.max}
-        IQR:#{iqr}
-        Outliers:#{outliers.inspect}
-        Variance:#{variance} 
-        Std Dev:#{std_dev}
-      DATA_STR
-    end
-  end  
+  VERSION = '1.3.0'
 end

data/lib/ministat/data.rb

@@ -0,0 +1,192 @@
+require 'mathn'
+
+module MiniStat
+  class Data
+    include Enumerable
+
+    attr_reader :data
+
+    def initialize(d)
+      @data   = d.map { |n| n.to_f }.sort
+      @sorted = true
+      clear_results
+    end
+
+    def <<(obj)
+      throw "#{obj.to_s} is not numeric" unless obj.to_f
+      @data << obj
+      # force computation!
+      clear_results
+      @sorted = false
+      obj
+    end
+
+    def each(&block)
+      @data.each(&block)
+    end
+
+    def clear_results
+      @q1, @q3, @iqr, @outliers, @std_dev, @variance  = nil
+      @mode, @harmonic_mean, @geometric_mean          = nil
+    end
+
+    ##
+    # Return the median of +data+. Naive implementaion
+    # -- does a sort on the data.
+    def median(data=@data)
+      unless @sorted and data == @data
+        data.sort!
+        @sorted = true
+      end
+      if data.size % 2 == 0
+        return (data[data.size / 2.0 - 1] + data[(data.size / 2.0)]) / 2.0
+      else 
+        return data[(data.size - 1)/2.0]
+      end
+    end
+
+    ##
+    # Partition a set of numbers about +pivot+
+    def partition(pivot, data=@data)
+      low  = []
+      high = []
+      data.each do |i|
+        high.push(i) if i > pivot
+        low.push(i)  if i < pivot
+      end
+      return {:low => low, :high => high}
+    end
+
+    ##
+    # A note on quartiles: the below methods DO NOT produce results
+    # that agree with R's default approach. There is no "one true"
+    # method for producing quartiles--R's default is that of S, the
+    # code below uses another (which I find simpler).  See:
+    # http://en.wikipedia.org/wiki/Quartile for details
+
+    ##
+    # First quartile.
+    def q1
+      @q1 ||= median(partition(median(@data), @data)[:low])
+    end
+
+    ##
+    # Third quartile
+    def q3
+      @q3 ||= median(partition(median(@data), @data)[:high])
+    end
+
+    ##
+    # Interquartile range, ie, the middle 50% of the data.
+    def iqr
+      @iqr ||= q3 - q1
+    end
+
+    ##
+    # Returns an array of outlying data points.
+    def outliers
+      @outliers ||= 
+        @data.map do |i|
+          i  if (i < q1 - (1.5 * iqr) or i > q3 + (1.5 * iqr))
+        end.compact
+    end
+
+    ##
+    # Computes arthmetic mean (most common average).
+    def mean(data=@data)
+      @mean = (data.inject(0) {|i,j| i += j}) / data.size
+    end
+
+    ##
+    # Computes mode and generates a histogram (for free!). 
+    # (We needed it anyway).
+    def mode
+      @hist     ||= {}
+      @max_freq ||= 0
+      @mode     ||= nil
+      unless @mode
+        @data.each do |val|
+          @hist[val] ||= 0
+          @hist[val] += 1
+          @max_freq, @mode = @hist[val], val if @hist[val] > @max_freq
+        end
+      end
+      @mode
+    end
+
+    ##
+    # Computes variance. Used to measure degree of spread 
+    # in dataset.
+    def variance
+      @variance ||= 
+        @data.inject(0) { |i,j| i += (j - mean(@data)) ** 2}  / (@data.size - 1)
+    end
+
+    ##
+    # Standard deviation. Square root of variance, measure of the
+    # spread of the data about the mean.
+    def std_dev
+      @std_dev ||= Math.sqrt(variance)
+    end
+
+    ##
+    # Geometric mean. Only applies to non-negative numbers, and
+    # relates to log-normal distribution.
+    def geometric_mean
+      if @data.any? { |x| x < 0 }
+        raise "Geometric mean only applies to non-negative data"
+      end
+
+      @geometric_mean ||= 
+        2 ** (mean @data.map { |x| Math.log2(x) })
+        # this overflowed for dataset with large numbers
+        # (@data.inject(1) {|i,j| i *= j})**(1.0/@data.size)
+    end
+
+    ##
+    # Harmonic or subcontrary mean. Tends strongly toward the least
+    # elements of the dataset.
+    def harmonic_mean
+      @harmonic_mean ||=
+        @data.size.to_f / (@data.inject(0) {|i,j| i += (1.0/j)})
+    end
+
+    ##
+    # Put the histogram into a string if we have it
+    def hist
+      if defined? @hist
+        # this is a textbook example of how to lie with statistics...
+        # TODO: iterate over a range rather than @hist.keys--a histogram
+        # produced out of the keys won't properly represent flat spots
+        # with no data. or something like that. do as i say, not as i do.
+        #
+        # this code borrows liberally from the ruby cookbook, recipe 5.12
+        # ORA, 2006
+        pairs = @hist.keys.map { |x| [x.to_s, @hist[x]] }.sort
+        largest_key_size = pairs.max { |x,y| x[0].size <=> y[0].size }[0].size
+        pairs.inject("") do |s,kv|
+        s<< "#{kv[0].ljust(largest_key_size)} |#{char*kv[1]}\n"
+      end
+      end
+    end
+
+    # Return a string with statisical info about a dataset.
+    def to_s
+      <<-DATA_STR    
+        Partition:#{partition(median).inspect} 
+        Mean:#{mean}
+        Geometric Mean:#{geometric_mean}
+        Harmonic Mean:#{harmonic_mean}
+        Median:#{median} 
+        Min:#{data.min} 
+        Q1:#{q1}
+        Q3:#{q3}
+        Max:#{data.max}
+        IQR:#{iqr}
+        Outliers:#{outliers.inspect}
+        Variance:#{variance} 
+        Std Dev:#{std_dev}
+      DATA_STR
+    end
+  end  
+end

data/test/data/3.dat

@@ -0,0 +1,200 @@
+5662
+9159
+9073
+6734
+4514
+57
+7188
+2928
+778
+9860
+3746
+5998
+9556
+4296
+6781
+3507
+9411
+7284
+1977
+1805
+2306
+4759
+7086
+5881
+7449
+3757
+2457
+6703
+2717
+5562
+7073
+3073
+775
+3854
+2509
+4039
+3083
+3485
+568
+2470
+8080
+1252
+4610
+2803
+1711
+8435
+5663
+9774
+4025
+3494
+5417
+9426
+2899
+5857
+3138
+6309
+806
+50
+1343
+3830
+6158
+115
+6471
+1667
+6483
+5617
+7975
+3454
+9927
+5623
+9245
+8514
+8999
+205
+8126
+2168
+5621
+5223
+3170
+488
+7466
+4310
+9697
+8166
+4740
+54
+1562
+5630
+2752
+6499
+5281
+7040
+2177
+8600
+1098
+946
+5274
+1211
+9808
+6266
+8639
+1664
+5376
+483
+9620
+2365
+4478
+6159
+6123
+6491
+2054
+9623
+2427
+5710
+3082
+3013
+9741
+7880
+9302
+3213
+22
+644
+9384
+8401
+3459
+1396
+5095
+6079
+5348
+578
+121
+5932
+3206
+8480
+6580
+6161
+3352
+7017
+7815
+9826
+6287
+7483
+6786
+1161
+599
+8235
+9864
+5665
+1381
+2241
+8478
+4958
+674
+1771
+7453
+6345
+5504
+6538
+5014
+1300
+3686
+4839
+7173
+6402
+3276
+2592
+8376
+8602
+926
+273
+6680
+4613
+5880
+1420
+9842
+2939
+2348
+1308
+8784
+2821
+7724
+7030
+3020
+79
+3876
+8801
+2376
+3309
+656
+5018
+4847
+7152
+7216
+9180
+2097
+2149
+3597
+3923
+5742
+9623

data/test/test_ministat.rb

@@ -1,20 +1,19 @@
 require 'rubygems'
-require 'minitest/unit'
-require 'ministat' 
+require 'minitest/autorun'
+require 'ministat'
 
 class TestMiniStat < MiniTest::Unit::TestCase
   def setup
     @data1 = [34, 47, 1, 15, 57, 24, 20, 11, 19, 50, 28, 37]
     @data2 = [60, 56, 61, 68, 51, 53, 69, 54]                  
-    @data3 = File.open('test/data/1.dat').map {|l| l.chomp}
-    @data4 = File.open('test/data/2.dat').map {|l| l.chomp}
+    @data3 = File.open('test/data/1.dat').map { |l| l.chomp }
+    @data4 = File.open('test/data/2.dat').map { |l| l.chomp }
+    @data5 = File.open('test/data/3.dat').map { |l| l.chomp }
     @ms1   = MiniStat::Data.new(@data1)
     @ms2   = MiniStat::Data.new(@data2)
     @ms3   = MiniStat::Data.new(@data3)
     @ms4   = MiniStat::Data.new(@data4)
-    # we test to within a tolerance to schluff off
-    # possible floating point and rounding errors
-    @error = 0.001 
+    @ms5   = MiniStat::Data.new(@data5)
   end
 
   def test_enum
@@ -24,20 +23,44 @@ class TestMiniStat < MiniTest::Unit::TestCase
     assert ms.mean == 1.5
   end
 
+  ##
+  # Expected test values are computed in a 3rd party statistics
+  # package (usually R) when possible.
+  #
+  # All the basic tests do a dummy check of adding a value to the data
+  # set and ensuring the the values are re-computed.
+
   def test_iqr
-    assert(@ms1.iqr - 25 < @error)
+    assert_in_delta 25, @ms1.iqr
+    assert_in_delta 11, @ms2.iqr
+    assert_in_delta 1.94, @ms3.iqr
+    assert_in_delta 9.15, @ms4.iqr
+    assert_in_delta 4677, @ms5.iqr
+    
+    @ms1 << 32
+    assert_in_delta 25, @ms1.iqr
   end                                                   
 
   def test_mean
-    assert(@ms2.mean - 59 < @error)
-    assert(@ms3.mean - 2.179 < @error)
-    assert(@ms4.mean - 1.878 < @error)
+    assert_in_delta 28.583, @ms1.mean
+    assert_in_delta 59, @ms2.mean
+    assert_in_delta 2.179, @ms3.mean
+    assert_in_delta 1.878, @ms4.mean
+    assert_in_delta 4884.695, @ms5.mean
+
+    @ms1 << 32
+    assert_in_delta 28.84615, @ms1.mean
   end
 
   def test_median
-    assert(@ms1.median - 26 < @error)
-    assert(@ms3.median - 1.735 < @error)
-    assert(@ms3.median - 2.8 < @error)
+    assert_in_delta 26, @ms1.median
+    assert_in_delta 58, @ms2.median
+    assert_in_delta 1.735, @ms3.median
+    assert_in_delta 2.8, @ms4.median
+    assert_in_delta 5016, @ms5.median
+    
+    @ms1 << 32
+    assert_in_delta 28, @ms1.median
   end
 
   def test_mode
@@ -48,34 +71,81 @@ class TestMiniStat < MiniTest::Unit::TestCase
   end
 
   def test_outliers
-    assert_equal(@ms1.outliers, [])
+    assert_equal @ms1.outliers, []
+    assert_includes @ms3.outliers, 6.0
+
+    @ms1 << 1000
+    assert_includes @ms1.outliers, 1000
   end
 
   def test_q1
-    assert(@ms1.q1 - 17 < @error)   
-    assert(@ms3.q1 - 1.05 < @error)
-    assert(@ms4.q1 - -2.4 < @error)
+    assert_in_delta 17, @ms1.q1
+    assert_in_delta 53.5, @ms2.q1
+    assert_in_delta 1.05, @ms3.q1
+    assert_in_delta(-2.4, @ms4.q1)
+    assert_in_delta 2442, @ms5.q1
+
+    # add below our initial q1 to skew the result
+    @ms1 << 1
+    assert_in_delta 13, @ms1.q1
   end
 
   def test_q3
-    assert(@ms1.q3 - 42 < @error)
-    assert(@ms3.q3 - 2.99 < @error)
-    assert(@ms4.q3 - 6.75 < @error)
+    assert_in_delta 42, @ms1.q3
+    assert_in_delta 64.5, @ms2.q3
+    assert_in_delta 2.99, @ms3.q3
+    assert_in_delta 6.75, @ms4.q3
+    assert_in_delta 7119, @ms5.q3
+
+    # add above our initial q3 to skew the result
+    @ms1 << 100
+    assert_in_delta 48.5, @ms1.q3
+  end
+
+  def test_variance
+    assert_in_delta 286.0833, @ms1.variance
+    assert_in_delta 45.71429, @ms2.variance
+    assert_in_delta 2.297878, @ms3.variance
+    assert_in_delta 57.06196, @ms4.variance
+    assert_in_delta 8376067, @ms5.variance, 1
+    
+    @ms1 << 32
+    assert_in_delta 263.141, @ms1.variance
   end
 
   def test_std_dev
-   assert(@ms2.std_dev - 6.324 < @error)
-   assert(@ms3.std_dev - 1.515 < @error)
-   assert(@ms4.std_dev - 7.553 < @error)
+    assert_in_delta 16.914, @ms1.std_dev
+    assert_in_delta 6.761, @ms2.std_dev
+    assert_in_delta 1.515, @ms3.std_dev
+    assert_in_delta 7.553, @ms4.std_dev
+    assert_in_delta 2894.144, @ms5.std_dev
+
+    @ms1 << 32
+    assert_in_delta 16.22162, @ms1.std_dev
   end
 
   def test_geo_mean
-    # debugger
+    assert_in_delta 58.66896, @ms2.geometric_mean
+    assert_in_delta 1.695651, @ms3.geometric_mean
+    # @ms4 contains negative numbers, so we should complain
+    assert_raises RuntimeError do 
+      @ms4.geometric_mean
+    end
+    assert_in_delta 3463.229, @ms5.geometric_mean
+
+    @ms1 << 32
+    assert_in_delta 21.63259, @ms1.geometric_mean
   end
 
   def test_harm_mean
-    # debugger
+    assert_in_delta 8.259642, @ms1.harmonic_mean
+    assert_in_delta 58.34724, @ms2.harmonic_mean
+    assert_in_delta 1.218216, @ms3.harmonic_mean
+    assert_in_delta 5.921447, @ms4.harmonic_mean
+    assert_in_delta 976.2331, @ms5.harmonic_mean
+
+    @ms1 << 32
+    assert_in_delta 8.759532, @ms1.harmonic_mean
   end
-end
 
-# Number of errors detected: 11
+end

metadata

@@ -1,121 +1,91 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: ministat
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 2
-  - 6
-  version: 1.2.6
+version: !ruby/object:Gem::Version
+  version: 1.3.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Dean Hudson
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-11-23 00:00:00 -08:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: rubyforge
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 0
-        - 4
-        version: 2.0.4
-  type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+date: 2011-10-16 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: minitest
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &70294540175900 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 0
-        version: "2.0"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: hoe
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *70294540175900
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: &70294540175380 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 6
-        - 2
-        version: 2.6.2
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
   type: :development
-  version_requirements: *id003
-description: |-
-  This is a simple package that generates simple statistical info on
-  numerical data sets of a single variable. It's nothing too fancy, but
+  prerelease: false
+  version_requirements: *70294540175380
+description: ! 'This is a simple package that generates simple statistical info on
+
+  numerical data sets of a single variable. It''s nothing too fancy, but
+
   maybe just enough to coat your numbers with a thin layer of science. Or,
-  at least to get you thinking about what it may take to do so.
-email: 
+
+  at least to get you thinking about what it may take to do so.'
+email:
 - dean@ero.com
-executables: 
+executables:
 - ministat
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - Manifest.txt
-files: 
+files:
 - History.rdoc
 - Manifest.txt
 - README.rdoc
 - Rakefile
 - bin/ministat
 - lib/ministat.rb
+- lib/ministat/data.rb
 - test/data/1.dat
 - test/data/2.dat
+- test/data/3.dat
 - test/test_ministat.rb
-has_rdoc: true
+- .gemtest
 homepage: http://github.com/deanh/MiniStat
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --main
 - README.rdoc
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: ministat
-rubygems_version: 1.3.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
-summary: This is a simple package that generates simple statistical info on numerical data sets of a single variable
-test_files: 
+summary: This is a simple package that generates simple statistical info on numerical
+  data sets of a single variable
+test_files:
 - test/test_ministat.rb