statsample-glm 0.0.1 → 0.1.0

data/lib/statsample-glm/glm/irls/logistic.rb

@@ -0,0 +1,46 @@
+require 'statsample-glm/glm/irls/base'
+
+module Statsample
+  module GLM
+    module IRLS
+      class Logistic < Statsample::GLM::IRLS::Base
+        def initialize data_set, dependent, opts={}
+          super data_set, dependent, opts
+        end
+
+        def to_s
+          "Statsample::GLM::Logistic"
+        end
+
+       protected
+
+        def measurement x, b
+          (x * b).map { |y| 1/(1 + Math.exp(-y)) }
+        end
+
+        def weight x, b
+          mus = measurement(x,b).column_vectors.map(&:to_a).flatten
+          mus_intermediate = mus.map { |p| 1 - p }
+          weights = mus.zip(mus_intermediate).collect { |x| x.inject(:*) }
+
+          w_mat = Matrix.I(weights.size)
+          w_enum = weights.to_enum
+          return w_mat.map do |x|
+            x.eql?(1) ? w_enum.next : x # diagonal consists of first derivatives of logit
+          end
+        end
+
+        def jacobian x, b, y
+          mu_flat     = measurement(x,b).column_vectors.map(&:to_a).flatten
+          column_data = y.zip(mu_flat).map { |x| x.inject(:-) }
+
+          x.transpose * Matrix.column_vector(column_data)
+        end
+
+        def hessian x, b
+          (x.transpose * weight(x, b) * x).map { |x| -x }
+        end
+      end
+    end
+  end
+end

data/lib/statsample-glm/glm/irls/poisson.rb

@@ -0,0 +1,48 @@
+require 'statsample-glm/glm/irls/base'
+
+module Statsample
+  module GLM
+    module IRLS
+      class Poisson < Statsample::GLM::IRLS::Base
+        def initialize data_set, dependent, opts={}
+          super data_set, dependent, opts
+        end
+
+        def to_s
+          puts "Logistic Regression (Statsample::Regression::GLM::Logistic)"
+        end
+       protected
+
+        def measurement x, b
+          if @opts[:link] == :log
+            (x * b).map { |y| Math.exp(y) }
+          elsif @opts[:link] == :sqrt
+            (x * b).map { |y| y**2 }
+          end
+        end
+
+        def weight x, b
+          m = measurement(x,b).column_vectors.map(&:to_a).flatten
+
+          w_mat  = Matrix.I(m.size)
+          w_enum = m.to_enum
+
+          return w_mat.map do |x|
+            x.eql?(1) ? w_enum.next : x # diagonal consists of first derivatives of logit
+          end
+        end
+
+        def hessian x, b
+          (x.transpose * weight(x, b) * x).map { |x| -x }
+        end
+
+        def jacobian x, b, y
+          measurement_flat = measurement(x,b).column_vectors.map(&:to_a).flatten
+          column_data = y.zip(measurement_flat).collect { |x| x.inject(:-) }
+
+          x.transpose * Matrix.columns([column_data])
+        end
+      end
+    end
+  end
+end

data/lib/statsample-glm/glm/logistic.rb

@@ -0,0 +1,16 @@
+require 'statsample-glm/glm/base'
+
+module Statsample
+  module GLM
+    class Logistic < Statsample::GLM::Base
+
+      def initialize data_set, dependent, opts
+        super data_set, dependent, opts
+      end
+      
+      def to_s
+        "Statsample::GLM::Logistic"
+      end
+    end
+  end
+end

data/lib/statsample-glm/glm/mle/base.rb

@@ -0,0 +1,157 @@
+module Statsample
+
+  module GLM
+    module MLE
+      class Base
+          attr_reader :coefficients, :iterations, 
+            :fitted_mean_values, :residuals, :degree_of_freedom,
+            :log_likelihood
+  
+        MIN_DIFF_PARAMETERS=1e-2
+
+        def initialize data_set, dependent, opts
+          @opts = opts
+
+          @data_set  = data_set
+          @dependent = dependent
+
+          @stop_criteria  = :parameters
+          @var_cov_matrix = nil
+          @iterations     = nil
+          @parameters     = nil
+
+          x = @data_set.to_matrix
+          y = @dependent.to_matrix(:vertical)
+
+          @coefficients   = newton_raphson x, y
+          @log_likelihood = _log_likelihood x, y, @coefficients
+          @fitted_mean_values = create_vector measurement(x, @coefficients).to_a.flatten
+          @residuals = @dependent - @fitted_mean_values
+          @degree_of_freedom  = @dependent.count - x.column_size
+
+          # This jugad is done because the last vector index for Normal is sigma^2
+          # which we dont want to return to the user.
+          @coefficients =  create_vector(self.is_a?(Statsample::GLM::MLE::Normal) ? 
+            @coefficients.to_a.flatten[0..-2] : @coefficients.to_a.flatten)
+        end
+
+        def standard_error
+          out = []
+
+          @data_set.fields.each_index do |i|
+            out << Math::sqrt(@var_cov_matrix[i,i])
+          end
+
+          out
+        end
+
+        # Newton Raphson with automatic stopping criteria.
+        # Based on: Von Tessin, P. (2005). Maximum Likelihood Estimation With Java and Ruby
+        #
+        # <tt>x</tt>:: matrix of dependent variables. Should have nxk dimensions
+        # <tt>y</tt>:: matrix of independent values. Should have nx1 dimensions
+        # <tt>@m</tt>:: class for @ming. Could be Normal or Logistic
+        # <tt>start_values</tt>:: matrix of coefficients. Should have 1xk dimensions
+        def newton_raphson(x,y, start_values=nil)
+          # deep copy?
+          if start_values.nil?
+              parameters = set_default_parameters(x)
+          else
+              parameters = start_values.dup
+          end
+          k = parameters.row_size
+
+          raise "n on y != n on x" if x.row_size != y.row_size
+          h  = nil
+          fd = nil
+
+          if @stop_criteria == :mle
+            old_likelihood = _log_likelihood(x, y, parameters)
+          else
+            old_parameters = parameters
+          end
+        
+          @opts[:iterations].times do |i|
+            @iterations = i + 1
+
+            h = second_derivative(x,y,parameters)
+            if h.singular?
+              raise "Hessian is singular!"
+            end
+            fd = first_derivative(x,y,parameters)
+            parameters = parameters - (h.inverse * (fd))
+            
+            if @stop_criteria == :parameters
+              flag = true
+              k.times do |j|
+                diff = ( parameters[j,0] - old_parameters[j,0] ) / parameters[j,0]
+                flag = false if diff.abs >= MIN_DIFF_PARAMETERS
+
+              end
+              
+              if flag
+                @var_cov_matrix = h.inverse*-1.0
+                return parameters
+              end
+              old_parameters = parameters
+            else
+              begin
+                new_likelihood = _log_likelihood(x,y,parameters)
+
+                if(new_likelihood < old_likelihood) or ((new_likelihood - old_likelihood) / new_likelihood).abs < @opts[:epsilon]
+                  @var_cov_matrix = h.inverse*-1.0
+                  break;
+                end
+                old_likelihood = new_likelihood
+              rescue =>e
+                puts "#{e}"
+              end
+            end
+          end
+          @parameters = parameters
+          parameters
+        end
+
+       private
+        # Calculate likelihood for matrices x and y, given b parameters
+        def likelihood x,y,b
+          prod = 1
+          x.row_size.times{|i|
+            xi=Matrix.rows([x.row(i).to_a.collect{|v| v.to_f}])
+            y_val=y[i,0].to_f
+            #fbx=f(b,x)
+            prod=prod*likelihood_i(xi, y_val ,b)
+          }
+          prod
+        end
+
+        # Calculate log likelihood for matrices x and y, given b parameters
+        def _log_likelihood x,y,b 
+          sum = 0
+          x.row_size.times{|i|
+            xi = Matrix.rows([x.row(i).to_a.collect{|v| v.to_f}])
+            y_val = y[i,0].to_f
+            sum += log_likelihood_i xi, y_val, b
+          }
+
+          sum
+        end
+        
+        # Creates a zero matrix Mx1, with M=x.M
+        def set_default_parameters x
+          fd = [0.0] * x.column_size
+
+          fd.push(0.1) if self.is_a? Statsample::GLM::MLE::Normal
+          Matrix.columns([fd])
+        end
+
+        def create_vector arr
+          Statsample::Vector.new(arr, :scale)
+        end
+      end
+    end
+  end
+end
+require 'statsample/mle/normal'
+require 'statsample/mle/logit'
+require 'statsample/mle/probit'

data/lib/statsample-glm/glm/mle/logistic.rb

@@ -0,0 +1,113 @@
+require 'statsample-glm/glm/mle/base'
+
+module Statsample
+  module GLM
+    module MLE
+      # Logistic MLE estimation. 
+      # See Statsample::Regression for methods to generate a logit regression.
+      # Usage:
+      # 
+      #   mle=Statsample::GLM::MLE::Logistic.new
+      #   mle.newton_raphson(x,y)
+      #   beta=mle.coefficients
+      #   likelihood=mle.likelihood(x, y, beta)
+      #   iterations=mle.iterations
+      #         
+      class Logistic < Statsample::GLM::MLE::Base
+
+       protected
+        # F(B'Xi)
+        def f(b,xi)
+          p_bx = (xi*b)[0,0] 
+          res  = (1.0/(1.0+Math::exp(-p_bx)))
+          if res == 0.0
+            res = 1e-15
+          elsif res == 1.0
+            res = 0.999999999999999
+          end
+    
+          res
+        end
+    
+        # Likehood for x_i vector, y_i scalar and b parameters
+        def likelihood_i(xi,yi,b)
+          (f(b,xi)**yi)*((1-f(b,xi))**(1-yi))
+        end
+    
+        # Log Likehood for x_i vector, y_i scalar and b parameters
+        def log_likelihood_i(xi,yi,b)
+          fbx = f(b,xi)
+          (yi.to_f*Math::log(fbx))+((1.0-yi.to_f)*Math::log(1.0-fbx))
+        end
+        
+        # First derivative of log-likelihood function
+        # x: Matrix (NxM)
+        # y: Matrix (Nx1)
+        # p: Matrix (Mx1)
+        def first_derivative(x,y,p)
+          raise "x.rows != y.rows"    if x.row_size != y.row_size
+          raise "x.columns != p.rows" if x.column_size != p.row_size            
+    
+          n  = x.row_size
+          k  = x.column_size
+          fd = Array.new(k)
+          k.times {|i| fd[i] = [0.0]}
+    
+          n.times do |i|
+            row    = x.row(i).to_a
+            value1 = (1 - y[i,0]) - p_plus(row,p)
+    
+            k.times do |j|
+              fd[j][0] -= value1*row[j]
+            end
+          end
+          Matrix.rows(fd, true)
+        end
+        # Second derivative of log-likelihood function
+        # x: Matrix (NxM)
+        # y: Matrix (Nx1)
+        # p: Matrix (Mx1)
+        def second_derivative(x,y,p2)
+          raise "x.rows!=y.rows" if x.row_size!=y.row_size
+          raise "x.columns!=p.rows" if x.column_size!=p2.row_size             
+          n = x.row_size
+          k = x.column_size
+          sd = Array.new(k)
+          k.times do |i|
+            arr = Array.new(k)
+            k.times{ |j| arr[j]=0.0}
+            sd[i] = arr
+          end
+          n.times do |i|
+            row = x.row(i).to_a
+            p_m = p_minus(row,p2)
+            k.times do |j|
+              k.times do |l|
+              sd[j][l] -= (p_m*(1-p_m)*row[j]*row[l])
+              end
+            end
+          end
+          Matrix.rows(sd, true)
+        end
+
+        def measurement x, b
+          (x * b).map { |y| 1/(1 + Math.exp(-y)) }
+        end
+        
+       private
+        def p_minus(x_row,p)
+          value = 0.0;
+          x_row.each_index { |i| value += x_row[i]*p[i,0]}
+          1/(1+Math.exp(-value))
+        end
+    
+        def p_plus(x_row,p)
+          value = 0.0;
+          x_row.each_index { |i| value += x_row[i]*p[i,0]}
+          1/(1+Math.exp(value))
+        end
+        
+        end # Logistic
+    end # MLE
+  end # GLM
+end # Statsample

data/lib/statsample-glm/glm/mle/normal.rb

@@ -0,0 +1,94 @@
+require 'statsample-glm/glm/mle/base'
+
+module Statsample
+  module GLM
+    module MLE
+      # Normal Distribution MLE estimation.
+      # Usage:
+      # TODO : Document this properly
+      #   mle=Statsample::MLE::Normal.new
+      #   mle.newton_raphson(x,y)
+      #   beta=mle.coefficients
+      #   likelihood=mle.likelihood(x,y,beta)
+      #   iterations=mle.iterations
+      class Normal < Statsample::GLM::MLE::Base
+
+       protected
+        def measurement data_set, coefficients
+          (data_set * coefficients[0..-2,0]).map { |xb| xb }
+        end
+        # Total MLE for given X, Y and B matrices (overridden over the Base version)
+        def _log_likelihood(x,y,b)
+          n      = x.row_size.to_f
+          sigma2 = b[b.row_size-1,0]
+          betas  = Matrix.columns([b.column(0). to_a[0...b.row_size-1]])
+          e      = y - (x * betas)
+          last   = (1 / (2*sigma2)) * e.t * e
+          (-(n / 2.0) * Math::log(2*Math::PI))-((n / 2.0)*Math::log(sigma2)) - last[0,0]
+        end
+        # First derivative for Normal Model.
+        # p should be [k+1,1], because the last parameter is sigma^2
+        def first_derivative(x,y,p)
+          raise "x.rows != y.rows" if x.row_size != y.row_size
+          raise "x.columns + 1 != p.rows" if x.column_size + 1 != p.row_size
+          
+          n = x.row_size
+          k = x.column_size
+          b = Array.new(k)
+
+          k.times{|i| b[i]=[p[i,0]]}
+          beta   = Matrix.rows(b)
+          sigma2 = p[k,0]
+          sigma4 = sigma2 * sigma2
+          e = y-(x * (beta))
+          xte = x.transpose*(e)
+          ete = e.transpose*(e)
+          #rows of the Jacobian
+          rows = Array.new(k+1)
+          k.times{|i| rows[i] = [xte[i,0] / sigma2]}
+          rows[k] = [ete[0,0] / (2*sigma4) - n / (2*sigma2)]
+          Matrix.rows(rows, true)
+        end
+        
+        # second derivative for normal model
+         # p should be [k+1,1], because the last parameter is sigma^2
+        def second_derivative(x,y,p)
+          raise "x.rows != y.rows"        if x.row_size != y.row_size
+          raise "x.columns + 1 != p.rows" if x.column_size + 1 != p.row_size
+          #n = x.row_size
+          k = x.column_size
+          b = Array.new(k)
+          k.times{|i| b[i] = [p[i,0]]}
+          beta = Matrix.rows(b)
+          sigma2 = p[k,0]
+          sigma4 = sigma2*sigma2
+          sigma6 = sigma2*sigma2*sigma2
+          e = y-(x*(beta))
+          xtx = x.transpose*(x)
+          xte = x.transpose*(e)
+          ete = e.transpose*(e)
+          #rows of the Hessian
+          rows = Array.new(k+1)
+
+          k.times do |i|
+            row = Array.new(k+1)
+            k.times do |j|
+                row[j] = -xtx[i,j] / sigma2
+            end
+            row[k] = -xte[i,0] / sigma4
+            rows[i] = row
+          end
+
+          last_row = Array.new(k+1)
+          k.times do |i|
+            last_row[i] = -xte[i,0] / sigma4
+          end
+
+          last_row[k] = 2*sigma4 - ete[0,0] / sigma6
+          rows[k] = last_row
+          Matrix.rows(rows, true)
+        end
+      end
+    end
+  end
+end