statsample 1.1.0 → 1.2.0

data/History.txt

@@ -1,3 +1,10 @@
+=== 1.2.0 / 2011-12-15
+  
+  * Added support for time series (TimeSeries object): MA, EMA, MACD, acf, lag and delta. [Rob Britton]
+  * Changed summary attribute to properly display 'b' value for simple linear regression [hstove]
+  * Merge pull request #6 from hstove/patch-1Changed summary attribute to properly display 'b' value for simple linear regression [Claudio Bustos]
+  * fix example code for CovariateMatrix [James Kebinger]
+
 === 1.1.0 / 2011-06-02
 
 * New Statsample::Anova::Contrast

data/Manifest.txt

@@ -102,6 +102,7 @@ lib/statsample/test/kolmogorovsmirnov.rb
 lib/statsample/test/levene.rb
 lib/statsample/test/t.rb
 lib/statsample/test/umannwhitney.rb
+lib/statsample/timeseries.rb
 lib/statsample/vector.rb
 lib/statsample/vector/gsl.rb
 po/es/statsample.mo
@@ -114,6 +115,7 @@ test/fixtures/correlation_matrix.rb
 test/fixtures/crime.txt
 test/fixtures/hartman_23.matrix
 test/fixtures/repeated_fields.csv
+test/fixtures/stock_data.csv
 test/fixtures/test_binomial.csv
 test/fixtures/test_csv.csv
 test/fixtures/test_xls.xls
@@ -156,6 +158,7 @@ test/test_stratified.rb
 test/test_test_f.rb
 test/test_test_kolmogorovsmirnov.rb
 test/test_test_t.rb
+test/test_tseries.rb
 test/test_umannwhitney.rb
 test/test_vector.rb
 test/test_xls.rb

data/README.txt

@@ -17,6 +17,7 @@ Include:
 * Regression: Simple, Multiple (OLS), Probit  and Logit
 * Factorial Analysis: Extraction (PCA and Principal Axis), Rotation (Varimax, Equimax, Quartimax) and Parallel Analysis and Velicer's MAP test, for estimation of number of factors.
 * Reliability analysis for simple scale and a DSL to easily analyze multiple scales using factor analysis and correlations, if you want it.
+* Basic time series support
 * Dominance Analysis, with multivariate dependent and bootstrap (Azen & Budescu)
 * Sample calculation related formulas
 * Structural Equation Modeling (SEM), using R libraries +sem+ and +OpenMx+
@@ -89,6 +90,7 @@ Include:
   * Statsample::Graph::Boxplot
   * Statsample::Graph::Histogram
   * Statsample::Graph::Scatterplot
+* Module Statsample::TimeSeries provides basic support for time series. 
 * Gem +statsample-sem+ provides a DSL to R libraries +sem+ and +OpenMx+
 * Close integration with gem <tt>reportbuilder</tt>, to easily create reports on text, html and rtf formats.
 

data/Rakefile

@@ -6,8 +6,10 @@ $:.unshift(File.dirname(__FILE__)+'/lib/')
 require 'rubygems'
 require 'statsample'
 require 'hoe'
-Hoe.plugin :git
+require 'rdoc'
 
+Hoe.plugin :git
+Hoe.plugin :doofus
 desc "Ruby Lint"
 task :lint do
   executable=Config::CONFIG['RUBY_INSTALL_NAME']
@@ -22,6 +24,9 @@ task :release do
 system %{git push origin master}
 end
 
+task "clobber_docs" do
+  # Only to omit warnings
+end
 desc "Update pot/po files."
 task "gettext:updatepo" do
   require 'gettext/tools'
@@ -40,7 +45,7 @@ h=Hoe.spec('statsample') do
   #self.testlib=:minitest
 	self.rubyforge_name = "ruby-statsample"
 	self.developer('Claudio Bustos', 'clbustos@gmail.com')
-	self.extra_deps << ["spreadsheet","~>0.6.5"] <<  ["reportbuilder", "~>1.4"] << ["minimization", "~>0.2.0"] << ["fastercsv", ">0"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", ">0"] << ["rserve-client", "~>0.2.5"] << ["rubyvis", "~>0.4.0"] << ["distribution", "~>0.3"]
+	self.extra_deps << ["spreadsheet","~>0.6.5"] <<  ["reportbuilder", "~>1.4"] << ["minimization", "~>0.2.0"] << ["fastercsv", ">0"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", ">0"] << ["rserve-client", "~>0.2.5"] << ["rubyvis", "~>0.5"] << ["distribution", "~>0.6"]
   
 	self.extra_dev_deps << ["hoe","~>0"] << ["shoulda","~>0"] << ["minitest", "~>2.0"] << ["rserve-client", "~>0"] << ["gettext", "~>0"] << ["mocha", "~>0"] << ["hoe-git", "~>0"]
   
@@ -67,6 +72,7 @@ source code first.
 	self.need_rdoc=false
 end
 
+if Rake.const_defined?(:RDocTask)
 Rake::RDocTask.new(:docs) do |rd|
   rd.main = h.readme_file
   rd.options << '-d' if (`which dot` =~ /\/dot/) unless
@@ -91,7 +97,7 @@ Rake::RDocTask.new(:docs) do |rd|
   end
 end
 
-
+end
 desc 'Publish rdocs with analytics support'
 task :publicar_docs => [:clean, :docs] do
   ruby %{agregar_adsense_a_doc.rb}

data/grab_references.rb

@@ -2,6 +2,7 @@
 require 'reportbuilder'
 refs=[]
 Dir.glob "**/*.rb" do |f|
+  next if f=~/pkg/
 	reference=false
 	File.open(f).each_line 	do |l|
 		

data/lib/statsample.rb

@@ -134,7 +134,7 @@ module Statsample
   
   create_has_library :gsl
   
-  VERSION = '1.1.0'
+  VERSION = '1.2.0'
   SPLIT_TOKEN = ","
   autoload(:Analysis, 'statsample/analysis')
   autoload(:Database, 'statsample/converters')
@@ -161,6 +161,7 @@ module Statsample
   autoload(:Test, 'statsample/test')
   autoload(:Factor, 'statsample/factor')
   autoload(:Graph, 'statsample/graph')
+  autoload(:TimeSeries, 'statsample/timeseries')
   
   
   class << self

data/lib/statsample/matrix.rb

@@ -266,16 +266,16 @@ module Statsample
     #   a=Matrix[[1.0, 0.3, 0.2],
     #            [0.3, 1.0, 0.5], 
     #            [0.2, 0.5, 1.0]]
-    #   a.extends CovariateMatrix
-    #   a.labels=%w{a b c}
-    #   a.submatrix(%{c a}, %w{b})
+    #   a.extend CovariateMatrix
+    #   a.fields=%w{a b c}
+    #   a.submatrix(%w{c a}, %w{b})
     #   => Matrix[[0.5],[0.3]]
-    #   a.submatrix(%{c a})
+    #   a.submatrix(%w{c a})
     #   => Matrix[[1.0, 0.2] , [0.2, 1.0]]
     def submatrix(rows,columns=nil)
       raise ArgumentError, "rows shouldn't be empty" if rows.respond_to? :size and rows.size==0
       columns||=rows
-      # Convert all labels on index
+      # Convert all fields on index
       row_index=rows.collect {|v| 
         r=v.is_a?(Numeric) ? v : fields_x.index(v)
         raise "Index #{v} doesn't exists on matrix" if r.nil?

data/lib/statsample/regression/simple.rb

@@ -110,7 +110,7 @@ module Statsample
             t.row [_("r"), f % r]
             t.row [_("r^2"), f % r2]
             t.row [_("a"), f % a]
-            t.row [_("b"), f % a]
+            t.row [_("b"), f % b]
             t.row [_("s.e"), f % standard_error]
           end
         end

data/lib/statsample/timeseries.rb

@@ -0,0 +1,169 @@
+module Statsample::TimeSeriesShorthands
+  # Creates a new Statsample::TimeSeries object
+  # Argument should be equal to TimeSeries.new
+  def to_time_series(*args)
+		Statsample::TimeSeries::TimeSeries.new(self, :scale, *args)
+	end
+
+  alias :to_ts :to_time_series
+end
+
+class Array
+  include Statsample::TimeSeriesShorthands
+end
+
+module Statsample
+  module TimeSeries
+    # Collection of data indexed by time.
+    # The order goes from earliest to latest.
+    class TimeSeries < Statsample::Vector
+      # Calculates the autocorrelation coefficients of the series.
+      #
+      # The first element is always 1, since that is the correlation
+      # of the series with itself.
+      #
+      # Usage:
+      #
+      #  ts = (1..100).map { rand }.to_time_series
+      #
+      #  ts.acf   # => array with first 21 autocorrelations
+      #  ts.acf 3 # => array with first 3 autocorrelations
+      #
+      def acf maxlags = nil
+        maxlags ||= (10 * Math.log10(size)).to_i
+
+        (0..maxlags).map do |i|
+          if i == 0
+            1.0
+          else
+            m = self.mean
+
+            # can't use Pearson coefficient since the mean for the lagged series should
+            # be the same as the regular series
+            ((self - m) * (self.lag(i) - m)).sum / self.variance_sample / (self.size - 1)
+          end
+        end
+      end
+
+      # Lags the series by k periods.
+      #
+      # The convention is to set the oldest observations (the first ones
+      # in the series) to nil so that the size of the lagged series is the
+      # same as the original.
+      #
+      # Usage:
+      #
+      #  ts = (1..10).map { rand }.to_time_series
+      #           # => [0.69, 0.23, 0.44, 0.71, ...]
+      #
+      #  ts.lag   # => [nil, 0.69, 0.23, 0.44, ...]
+      #  ts.lag 2 # => [nil, nil, 0.69, 0.23, ...]
+      #
+      def lag k = 1
+        return self if k == 0
+
+        dup.tap do |lagged|
+          (lagged.size - 1).downto k do |i|
+            lagged[i] = lagged[i - k]
+          end
+
+          (0...k).each do |i|
+            lagged[i] = nil
+          end
+          lagged.set_valid_data
+        end
+      end
+
+      # Performs a first difference of the series.
+      #
+      # The convention is to set the oldest observations (the first ones
+      # in the series) to nil so that the size of the diffed series is the
+      # same as the original.
+      #
+      # Usage:
+      #
+      #  ts = (1..10).map { rand }.to_ts
+      #            # => [0.69, 0.23, 0.44, 0.71, ...]
+      #
+      #  ts.diff   # => [nil, -0.46, 0.21, 0.27, ...]
+      #  
+      def diff
+        self - self.lag
+      end
+
+      # Calculates a moving average of the series using the provided
+      # lookback argument. The lookback defaults to 10 periods.
+      #
+      # Usage:
+      #
+      #   ts = (1..100).map { rand }.to_ts
+      #            # => [0.69, 0.23, 0.44, 0.71, ...]
+      #
+      #   # first 9 observations are nil
+      #   ts.ma    # => [ ... nil, 0.484... , 0.445... , 0.513 ... , ... ]
+      def ma n = 10
+        return mean if n >= size
+
+        ([nil] * (n - 1) + (0..(size - n)).map do |i|
+          self[i...(i + n)].inject(&:+) / n
+        end).to_time_series
+      end
+
+      # Calculates an exponential moving average of the series using a
+      # specified parameter. If wilder is false (the default) then the EMA
+      # uses a smoothing value of 2 / (n + 1), if it is true then it uses the
+      # Welles Wilder smoother of 1 / n.
+      #
+      # Warning for EMA usage: EMAs are unstable for small series, as they
+      # use a lot more than n observations to calculate. The series is stable
+      # if the size of the series is >= 3.45 * (n + 1)
+      #
+      # Usage: 
+      #
+      #   ts = (1..100).map { rand }.to_ts
+      #            # => [0.69, 0.23, 0.44, 0.71, ...]
+      #
+      #   # first 9 observations are nil
+      #   ts.ema   # => [ ... nil, 0.509... , 0.433..., ... ]
+      def ema n = 10, wilder = false
+        smoother = wilder ? 1.0 / n : 2.0 / (n + 1)
+
+        # need to start everything from the first non-nil observation
+        start = self.data.index { |i| i != nil }
+
+        # first n - 1 observations are nil
+        base = [nil] * (start + n - 1)
+
+        # nth observation is just a moving average
+        base << self[start...(start + n)].inject(0.0) { |s, a| a.nil? ? s : s + a } / n
+
+        (start + n).upto size - 1 do |i|
+          base << self[i] * smoother + (1 - smoother) * base.last
+        end
+
+        base.to_time_series
+      end
+
+      # Calculates the MACD (moving average convergence-divergence) of the time
+      # series - this is a comparison of a fast EMA with a slow EMA.
+      def macd fast = 12, slow = 26, signal = 9
+        series = ema(fast) - ema(slow)
+        [series, series.ema(signal)]
+      end
+
+      # Borrow the operations from Vector, but convert to time series
+      def + series
+        super.to_a.to_ts
+      end
+
+      def - series
+        super.to_a.to_ts
+      end
+
+      def to_s
+        sprintf("Time Series(type:%s, n:%d)[%s]", @type.to_s, @data.size,
+                @data.collect{|d| d.nil? ? "nil":d}.join(","))
+      end
+    end
+  end
+end

data/test/fixtures/stock_data.csv

@@ -0,0 +1,500 @@
+17.66
+17.65
+17.68
+17.66
+17.68
+17.67
+17.68
+17.68
+17.67
+17.67
+17.68
+17.71
+17.74
+17.72
+17.73
+17.76
+17.74
+17.69
+17.69
+17.67
+17.66
+17.67
+17.69
+17.69
+17.68
+17.65
+17.65
+17.64
+17.63
+17.64
+17.67
+17.68
+17.7
+17.68
+17.69
+17.69
+17.72
+17.71
+17.71
+17.71
+17.69
+17.69
+17.71
+17.72
+17.71
+17.68
+17.68
+17.68
+17.69
+17.68
+17.68
+17.69
+17.67
+17.69
+17.71
+17.7
+17.7
+17.71
+17.73
+17.74
+17.74
+17.74
+17.76
+17.77
+17.55
+17.55
+17.5
+17.46
+17.49
+17.54
+17.51
+17.54
+17.57
+17.54
+17.52
+17.53
+17.56
+17.55
+17.55
+17.54
+17.55
+17.55
+17.55
+17.54
+17.52
+17.53
+17.51
+17.52
+17.5
+17.5
+17.5
+17.49
+17.46
+17.47
+17.48
+17.45
+17.41
+17.39
+17.38
+17.43
+17.44
+17.43
+17.43
+17.46
+17.46
+17.47
+17.47
+17.45
+17.48
+17.49
+17.5
+17.49
+17.48
+17.49
+17.47
+17.47
+17.44
+17.44
+17.43
+17.45
+17.42
+17.43
+17.43
+17.44
+17.44
+17.43
+17.41
+17.41
+17.38
+17.38
+17.37
+17.37
+17.37
+17.3
+17.28
+17.27
+17.19
+16.41
+16.44
+16.48
+16.53
+16.51
+16.57
+16.54
+16.59
+16.64
+16.6
+16.65
+16.69
+16.69
+16.68
+16.64
+16.65
+16.66
+16.64
+16.61
+16.65
+16.67
+16.66
+16.65
+16.61
+16.59
+16.57
+16.55
+16.55
+16.57
+16.54
+16.6
+16.62
+16.6
+16.59
+16.61
+16.66
+16.69
+16.67
+16.65
+16.66
+16.65
+16.65
+16.68
+16.68
+16.67
+16.64
+16.73
+16.76
+16.75
+16.79
+16.8
+16.77
+16.74
+16.76
+16.83
+16.84
+16.82
+16.89
+16.93
+16.94
+16.9
+16.92
+16.88
+16.85
+16.87
+16.8
+16.79
+16.85
+16.85
+16.8
+16.82
+16.85
+16.9
+16.86
+16.79
+16.75
+16.78
+17.06
+17.05
+17.04
+17.02
+17.01
+17.02
+17.05
+17.07
+17.08
+17.09
+17.1
+17.11
+17.09
+17.1
+17.1
+17.12
+17.17
+17.16
+17.17
+17.18
+17.18
+17.18
+17.17
+17.15
+17.14
+17.13
+17.14
+17.13
+17.12
+17.12
+17.09
+17.09
+17.11
+17.06
+17.07
+17.06
+17.07
+17.06
+17.09
+17.05
+17.04
+17.04
+16.99
+17
+17.03
+17
+16.97
+16.96
+16.98
+16.98
+16.98
+17.03
+17
+17
+17
+17.02
+17
+17.02
+17.01
+17.02
+17.03
+17.03
+17.01
+17.03
+17.03
+17.03
+17.01
+17.03
+17.05
+17.05
+17.08
+17.04
+17.01
+17.03
+17.02
+17.03
+17.04
+17.05
+17.37
+17.35
+17.34
+17.32
+17.29
+17.29
+17.22
+17.26
+17.3
+17.34
+17.33
+17.39
+17.4
+17.39
+17.48
+17.5
+17.47
+17.43
+17.4
+17.42
+17.46
+17.48
+17.48
+17.46
+17.46
+17.45
+17.43
+17.44
+17.48
+17.43
+17.45
+17.47
+17.46
+17.46
+17.48
+17.48
+17.48
+17.46
+17.5
+17.55
+17.58
+17.57
+17.56
+17.59
+17.61
+17.62
+17.63
+17.62
+17.61
+17.61
+17.62
+17.64
+17.65
+17.61
+17.62
+17.66
+17.65
+17.64
+17.63
+17.64
+17.64
+17.64
+17.63
+17.61
+17.61
+17.62
+17.63
+17.64
+17.65
+17.66
+17.68
+17.69
+17.69
+17.69
+17.66
+17.69
+17.69
+17.62
+17.68
+17.64
+17.65
+17.61
+17.52
+17.56
+17.55
+17.55
+17.48
+17.45
+17.46
+17.46
+17.44
+17.47
+17.5
+17.49
+17.5
+17.53
+17.53
+17.54
+17.51
+17.51
+17.53
+17.53
+17.53
+17.55
+17.55
+17.54
+17.56
+17.59
+17.57
+17.58
+17.58
+17.57
+17.59
+17.57
+17.55
+17.51
+17.51
+17.52
+17.52
+17.53
+17.55
+17.59
+17.61
+17.61
+17.6
+17.6
+17.62
+17.65
+17.62
+17.6
+17.6
+17.62
+17.61
+17.62
+17.63
+17.64
+17.65
+17.61
+17.62
+17.64
+17.63
+17.62
+17.6
+17.57
+17.57
+17.6
+17.59
+17.6
+17.61
+17.61
+17.63
+17.63
+17.59
+17.58
+17.76
+17.79
+17.76
+17.73
+17.74
+17.73
+17.67
+17.66
+17.66
+17.64
+17.63
+17.62
+17.61
+17.6
+17.61
+17.61
+17.6
+17.6
+17.64
+17.65
+17.65
+17.63
+17.61
+17.6
+17.63
+17.63
+17.62
+17.63
+17.64
+17.62
+17.63
+17.65
+17.64
+17.6
+17.59
+17.59
+17.58
+17.58
+17.6
+17.6
+17.6
+17.6
+17.6
+17.58
+17.59
+17.6
+17.6
+17.6
+17.59
+17.59
+17.58
+17.58
+17.65
+17.65