statsample 0.3.4 → 0.4.0

data/lib/statsample/mle.rb

@@ -0,0 +1,137 @@
+module Statsample
+    # Module for MLE calculations.
+    # Use subclass of BaseMLE for specific MLE model estimation.
+    # Usage:
+    # 
+    #   mle=Statsample::MLE::Probit.new
+    #   mle.newton_raphson(x,y)
+    #   beta=mle.parameters
+    #   likehood=mle.likehood(x,y,beta)
+    #   iterations=mle.iterations
+    # 
+    module MLE
+        class BaseMLE
+        attr_accessor :verbose
+        attr_accessor :output
+        # Could be :parameters or :mle
+        attr_accessor :stop_criteria
+        # Variance - Covariance matrix
+        attr_reader :var_cov_matrix
+        # Iterations
+        attr_reader :iterations
+        # Parameters (beta coefficients)
+        attr_reader :parameters
+        ITERATIONS=100
+        MIN_DIFF=1e-5
+        MIN_DIFF_PARAMETERS=1e-2
+        # Model should be a MLE subclass
+        def initialize()
+            @verbose        = false
+            @output         = STDOUT
+            @stop_criteria  = :parameters
+            @var_cov_matrix = nil
+            @iterations     = nil
+            @parameters     = nil
+        end
+        # Calculate likehood for matrices x and y, given b parameters
+        def likehood(x,y,b)
+            prod=1
+            x.row_size.times{|i|
+                xi=Matrix.rows([x.row(i).to_a.collect{|v| v.to_f}])
+                y_val=y[i,0].to_f
+                fbx=f(b,x)
+                prod=prod*likehood_i(xi,y_val,b)
+            }
+            prod
+        end
+        # Calculate log likehood for matrices x and y, given b parameters
+        def log_likehood(x,y,b)
+            sum=0
+            x.row_size.times{|i|
+                xi=Matrix.rows([x.row(i).to_a.collect{|v| v.to_f}])
+                y_val=y[i,0].to_f
+                sum+=log_likehood_i(xi,y_val,b)
+            }
+            sum
+        end
+        # Creates a zero matrix Mx1, with M=x.M
+        def set_default_parameters(x)
+            fd=x.column_size.times.collect{|i| 0.0}
+            fd.push(0.1)    if self.is_a? Statsample::MLE::Normal
+            parameters = Matrix.columns([fd])
+        end
+        
+        # Newton Raphson with automatic stopping criteria.
+        # Based on: Von Tessin, P. (2005). Maximum Likelihood Estimation With Java and Ruby
+        #
+        # <tt>x</tt>:: matrix of dependent variables. Should have nxk dimensions
+        # <tt>y</tt>:: matrix of independent values. Should have nx1 dimensions
+        # <tt>@m</tt>:: class for @ming. Could be Normal or Logit
+        # <tt>start_values</tt>:: matrix of coefficients. Should have 1xk dimensions
+        def newton_raphson(x,y, start_values=nil)
+            # deep copy?
+            if start_values.nil?
+                parameters=set_default_parameters(x)
+            else
+                parameters = start_values.dup
+            end
+            k=parameters.row_size
+            cv=Matrix.rows([([1.0]*k)])
+            last_diff=nil
+            raise "n on y != n on x" if x.row_size!=y.row_size
+            h=nil
+            fd=nil
+            if @stop_criteria==:mle
+                old_likehood=log_likehood(x, y, parameters)
+            else
+                old_parameters=parameters
+            end
+            ITERATIONS.times do |i|
+                @iterations=i+1
+                puts "Set #{i}" if @verbose
+                h = second_derivative(x,y,parameters)
+                if h.singular?
+                    raise "Hessian is singular!"
+                end
+                fd = first_derivative(x,y,parameters)
+                parameters = parameters-(h.inverse*(fd))
+
+                if @stop_criteria==:parameters
+                    flag=true
+                    k.times do |j|
+                        diff= ( parameters[j,0] - old_parameters[j,0] ) / parameters[j,0]
+                        flag=false if diff.abs >= MIN_DIFF_PARAMETERS
+                        @output.puts "Parameters #{j}: #{diff}" if @verbose
+                    end
+
+                    if flag
+                        @var_cov_matrix = h.inverse*-1.0
+                        return parameters
+                    end
+                    old_parameters=parameters
+                else
+                    begin
+                        new_likehood = log_likehood(x,y,parameters)
+                        @output.puts "[#{i}]Log-MLE:#{new_likehood} (Diff:#{(new_likehood-old_likehood) / new_likehood})" if @verbose
+                        if(new_likehood < old_likehood) or ((new_likehood - old_likehood) / new_likehood).abs < MIN_DIFF
+                            @var_cov_matrix = h.inverse*-1.0
+                        #@output.puts "Ok"
+                            break;
+                        end
+                        old_likehood=new_likehood
+                    rescue =>e
+                        puts "#{e}"
+                        #puts "dup"
+                    end
+                end
+            end
+            @parameters=parameters
+            parameters
+        end
+    end
+    end
+end
+
+require 'statsample/mle/normal'
+require 'statsample/mle/logit'
+require 'statsample/mle/probit'

data/lib/statsample/mle/logit.rb

@@ -0,0 +1,95 @@
+module Statsample
+    module MLE
+        # Logit MLE estimation.
+        # Usage:
+        # 
+        #   mle=Statsample::MLE::Logit.new
+        #   mle.newton_raphson(x,y)
+        #   beta=mle.parameters
+        #   likehood=mle.likehood(x,y,beta)
+        #   iterations=mle.iterations
+        #         
+        class Logit < BaseMLE
+        # F(B'Xi)
+        def f(b,xi)
+            p_bx=(xi*b)[0,0] 
+            res=(1.0/(1.0+Math::exp(-p_bx)))
+            if res==0.0
+                res=1e-15
+            elsif res==1.0
+                res=0.999999999999999
+            end
+            res
+        end
+        # Likehood for x_i vector, y_i scalar and b parameters
+        def likehood_i(xi,yi,b)
+            (f(b,xi)**yi)*((1-f(b,xi))**(1-yi))
+        end
+        # Log Likehood for x_i vector, y_i scalar and b parameters
+        def log_likehood_i(xi,yi,b)
+            fbx=f(b,xi)
+            (yi.to_f*Math::log(fbx))+((1.0-yi.to_f)*Math::log(1.0-fbx))
+        end
+        
+        # First derivative of log-likehood function
+        # x: Matrix (NxM)
+        # y: Matrix (Nx1)
+        # p: Matrix (Mx1)
+        def first_derivative(x,y,p)
+                raise "x.rows!=y.rows" if x.row_size!=y.row_size
+                raise "x.columns!=p.rows" if x.column_size!=p.row_size            
+            n = x.row_size
+            k = x.column_size
+            fd = Array.new(k)
+            k.times {|i| fd[i] = [0.0]}
+            n.times do |i|
+	            row = x.row(i).to_a
+	            value1 = (1-y[i,0]) -p_plus(row,p)
+            k.times do |j|
+	            fd[j][0] -= value1*row[j]
+	            end
+            end
+            Matrix.rows(fd, true)
+
+        end
+        # Second derivative of log-likehood function
+        # x: Matrix (NxM)
+        # y: Matrix (Nx1)
+        # p: Matrix (Mx1)
+        def second_derivative(x,y,p)
+                raise "x.rows!=y.rows" if x.row_size!=y.row_size
+                raise "x.columns!=p.rows" if x.column_size!=p.row_size             
+            n = x.row_size
+            k = x.column_size
+            sd = Array.new(k)
+            k.times do |i|
+                arr = Array.new(k)
+                k.times{ |j| arr[j]=0.0}
+                sd[i] = arr
+            end
+            n.times do |i|
+                row = x.row(i).to_a
+                p_m = p_minus(row,p)
+                k.times do |j|
+                    k.times do |l|
+                    sd[j][l] -= p_m *(1-p_m)*row[j]*row[l]
+                    end
+                end
+            end
+            Matrix.rows(sd, true)
+        end
+        private
+        def p_minus(x_row,p)
+            value = 0.0;
+            x_row.each_index { |i| value += x_row[i]*p[i,0]}
+            1/(1+Math.exp(-value))
+        end
+        def p_plus(x_row,p)
+            value = 0.0;
+            x_row.each_index { |i| value += x_row[i]*p[i,0]}
+            1/(1+Math.exp(value))
+        end
+
+        end # Logit
+    end # MLE
+end # Statsample

data/lib/statsample/mle/normal.rb

@@ -0,0 +1,83 @@
+module Statsample
+    module MLE
+        # Normal Distribution MLE estimation.
+        # Usage:
+        # 
+        #   mle=Statsample::MLE::Normal.new
+        #   mle.newton_raphson(x,y)
+        #   beta=mle.parameters
+        #   likehood=mle.likehood(x,y,beta)
+        #   iterations=mle.iterations
+
+        class Normal < BaseMLE
+            # Total MLE for given X, Y and B matrices
+            def log_likehood(x,y,b)
+                n=x.row_size.to_f
+                sigma2=b[b.row_size-1,0]
+                betas=Matrix.columns([b.column(0). to_a[0...b.row_size-1]])
+                e=y-(x*betas)
+                last=(1 / (2*sigma2))*e.t*e
+                (-(n / 2.0) * Math::log(2*Math::PI))-((n / 2.0)*Math::log(sigma2)) - last[0,0]
+            end
+            # First derivative for Normal Model.
+            # p should be [k+1,1], because the last parameter is sigma^2
+            def first_derivative(x,y,p)
+                raise "x.rows!=y.rows" if x.row_size!=y.row_size
+                raise "x.columns+1!=p.rows" if x.column_size+1!=p.row_size
+                
+                n = x.row_size
+                k = x.column_size
+                b = Array.new(k)
+                k.times{|i| b[i]=[p[i,0]]}
+                beta = Matrix.rows(b)
+                sigma2 = p[k,0]
+                sigma4=sigma2*sigma2
+                e = y-(x*(beta))
+                xte = x.transpose*(e)
+                ete = e.transpose*(e)
+                #rows of the Jacobian
+                rows = Array.new(k+1)
+                k.times{|i| rows[i] = [xte[i,0] / sigma2]}
+                rows[k] = [ete[0,0] / (2*sigma4) - n / (2*sigma2)]
+                fd = Matrix.rows(rows, true)
+            end
+            
+            # second derivative for normal model
+             # p should be [k+1,1], because the last parameter is sigma^2
+            def second_derivative(x,y,p)
+                raise "x.rows!=y.rows" if x.row_size!=y.row_size
+                raise "x.columns+1!=p.rows" if x.column_size+1!=p.row_size
+
+                n = x.row_size
+                k = x.column_size
+                b = Array.new(k)
+                k.times{|i| b[i]=[p[i,0]]}
+                beta = Matrix.rows(b)
+                sigma2 = p[k,0]
+                sigma4=sigma2*sigma2
+                sigma6 = sigma2*sigma2*sigma2
+                e = y-(x*(beta))
+                xtx = x.transpose*(x)
+                xte = x.transpose*(e)
+                ete = e.transpose*(e)
+                #rows of the Hessian
+                rows = Array.new(k+1)
+                k.times do |i|
+                    row = Array.new(k+1)
+                    k.times do |j|
+                        row[j] = -xtx[i,j] / sigma2
+                    end
+                    row[k] = -xte[i,0] / sigma4
+                    rows[i] = row
+                end
+                last_row = Array.new(k+1)
+                k.times do |i|
+                    last_row[i] = -xte[i,0] / sigma4
+                end
+                last_row[k] = 2*sigma4 - ete[0,0] / sigma6
+                rows[k] = last_row
+                sd = Matrix.rows(rows, true)
+            end
+        end
+    end
+end

data/lib/statsample/mle/probit.rb

@@ -0,0 +1,93 @@
+require 'matrix_extension'
+module Statsample
+    module MLE
+        # Probit MLE estimation.
+        # Usage:
+        # 
+        #   mle=Statsample::MLE::Probit.new
+        #   mle.newton_raphson(x,y)
+        #   beta=mle.parameters
+        #   likehood=mle.likehood(x,y,beta)
+        #   iterations=mle.iterations
+        class Probit < BaseMLE
+        # F(B'Xi)
+        if  HAS_GSL
+            # F(B'Xi)
+            def f(b,x)
+                p_bx=(x*b)[0,0] 
+                GSL::Cdf::ugaussian_P(p_bx)
+            end
+            # f(B'Xi)
+            def ff(b,x)
+                p_bx=(x*b)[0,0] 
+                GSL::Ran::ugaussian_pdf(p_bx)
+            end
+        else
+            def f(b,x) #:nodoc:
+                p_bx=(x*b)[0,0] 
+                Distribution::Normal.cdf(p_bx)
+            end
+            def ff(b,x) #:nodoc:
+                p_bx=(x*b)[0,0] 
+                Distribution::Normal.pdf(p_bx)
+                
+            end
+        end
+        # Log Likehood for x_i vector, y_i scalar and b parameters
+        def log_likehood_i(xi,yi,b)
+            fbx=f(b,xi)
+            (yi.to_f*Math::log(fbx))+((1.0-yi.to_f)*Math::log(1.0-fbx))
+        end
+        # First derivative of log-likehood probit function
+        # x: Matrix (NxM)
+        # y: Matrix (Nx1)
+        # p: Matrix (Mx1)
+        def first_derivative(x,y,b)
+                raise "x.rows!=y.rows" if x.row_size!=y.row_size
+                raise "x.columns!=p.rows" if x.column_size!=b.row_size            
+            n = x.row_size
+            k = x.column_size
+            fd = Array.new(k)
+            k.times {|i| fd[i] = [0.0]}
+            n.times do |i|
+	            xi = Matrix.rows([x.row(i).to_a])
+	            fbx=f(b,xi)
+                value1 = (y[i,0]-fbx)/ ( fbx*(1-fbx))*ff(b,xi) 
+            k.times do |j|
+	            fd[j][0] += value1*xi[0,j]
+	            end
+            end
+            Matrix.rows(fd, true)
+        end
+        # Second derivative of log-likehood probit function
+        # x: Matrix (NxM)
+        # y: Matrix (Nx1)
+        # p: Matrix (Mx1)
+
+        def second_derivative(x,y,b)
+            raise "x.rows!=y.rows" if x.row_size!=y.row_size
+            raise "x.columns!=p.rows" if x.column_size!=b.row_size
+            n = x.row_size
+            k = x.column_size
+            if HAS_GSL
+                sum=GSL::Matrix.zeros(k)
+            else
+                sum=Matrix.zero(k)
+            end
+            n.times do |i|
+                xi=Matrix.rows([x.row(i).to_a])
+                fbx=f(b,xi)
+                val=((ff(b,xi)**2) / (fbx*(1.0-fbx)))*xi.t*xi
+                if HAS_GSL
+                    val=val.to_gsl
+                end
+                sum-=val
+            end
+            if HAS_GSL
+                sum=sum.to_matrix
+            end
+            sum
+        end
+        end # Probit
+    end # MLE
+end # Statsample

data/lib/statsample/regression.rb

@@ -1,9 +1,11 @@
 require 'statsample/regression/simple'
-require 'statsample/regression/logit'
 require 'statsample/regression/multiple'
 require 'statsample/regression/multiple/alglibengine'
 require 'statsample/regression/multiple/rubyengine'
 require 'statsample/regression/multiple/gslengine'
+require 'statsample/regression/binomial'
+require 'statsample/regression/binomial/logit'
+require 'statsample/regression/binomial/probit'
 
 module Statsample
     # Module for regression procedures 

data/lib/statsample/regression/binomial.rb

@@ -0,0 +1,65 @@
+
+module Statsample
+    module Regression
+        module Binomial
+            def self.logit(ds,y_var)
+                Logit.new(ds,y_var)                
+            end
+            def self.probit(ds,y_var)
+                Probit.new(ds,y_var)                
+            end
+            
+         class BaseEngine
+             attr_reader :log_likehood, :iterations
+            def initialize(ds,y_var,model)
+                @ds=ds
+                @y_var=y_var
+                @dy=@ds[@y_var]
+                @ds_indep=ds.dup(ds.fields-[y_var])
+                constant=([1.0]*ds.cases).to_vector(:scale)
+                @ds_indep.add_vector("_constant",constant)
+                mat_x=@ds_indep.to_matrix
+                mat_y=@dy.to_matrix(:vertical)
+                @fields=@ds_indep.fields
+                @model=model
+                coeffs=model.newton_raphson(mat_x, mat_y)
+                @coeffs=assign_names(coeffs.column(0).to_a)
+                @iterations=model.iterations
+                @var_cov_matrix=model.var_cov_matrix
+                @log_likehood=model.log_likehood(mat_x, mat_y, coeffs)
+                end # init
+                # Coefficients standard error
+                def coeffs_se
+                    out={}
+                    @fields.each_index{|i|
+                        f=@fields[i]
+                        out[f]=Math::sqrt(@var_cov_matrix[i,i])
+                    }
+                    out.delete("_constant")
+                    out
+                end
+                def constant
+                    @coeffs['_constant']
+                end
+                def coeffs
+                    c=@coeffs.dup
+                    c.delete("_constant")
+                    c
+                end
+                # Constant standard error
+                def constant_se
+                    i=@fields.index :_constant
+                    Math::sqrt(@var_cov_matrix[i,i])
+                end
+                def assign_names(c)
+                    a={}
+                    @fields.each_index {|i|
+                        a[@fields[i]]=c[i]
+                    }
+                    a
+                end
+            end # Base Engine
+            
+        end # Dichotomic
+    end # Regression
+end # Stasample