statsample-ekatena 2.0.2

data/lib/statsample-ekatena/regression/multiple/matrixengine.rb

@@ -0,0 +1,205 @@
+module Statsample
+module Regression
+module Multiple
+  # Pure Ruby Class for Multiple Regression Analysis, based on a covariance or correlation matrix.
+  #
+  # Use Statsample::Regression::Multiple::RubyEngine if you have a 
+  # Dataset, to avoid setting all details.
+  # 
+  # <b>Remember:</b> NEVER use a Covariance data if you have missing data. Use only correlation matrix on that case.
+  #
+  # 
+  # Example:
+  #
+  #   matrix=[[1.0, 0.5, 0.2], [0.5, 1.0, 0.7], [0.2, 0.7, 1.0]]
+  #   
+  #   lr=Statsample::Regression::Multiple::MatrixEngine.new(matrix,2)
+
+class MatrixEngine < BaseEngine 
+  # Hash of standard deviation of predictors. 
+  # Only useful for Correlation Matrix, because by default is set to 1
+  attr_accessor :x_sd
+  # Standard deviation of criterion
+  # Only useful for Correlation Matrix, because by default is set to 1
+  attr_accessor :y_sd
+  # Hash of mean for predictors. By default, set to 0
+  attr_accessor :x_mean
+  
+  # Mean for criteria. By default, set to 0
+  attr_accessor :y_mean
+  
+  # Number of cases
+  attr_writer :cases
+  attr_writer :digits
+  # Create object
+  #
+  def initialize(matrix,y_var, opts=Hash.new)
+    matrix.extend Statsample::CovariateMatrix
+    raise "#{y_var} variable should be on data" unless matrix.fields.include? y_var
+    if matrix._type==:covariance
+      @matrix_cov=matrix
+      @matrix_cor=matrix.correlation
+      @no_covariance=false
+    else
+      @matrix_cor=matrix
+      @matrix_cov=matrix
+      @no_covariance=true
+    end
+    
+    @y_var=y_var
+    @fields=matrix.fields-[y_var]
+    
+    @n_predictors=@fields.size
+    @predictors_n=@n_predictors
+    @matrix_x= @matrix_cor.submatrix(@fields)
+    @matrix_x_cov= @matrix_cov.submatrix(@fields)
+    raise LinearDependency, "Regressors are linearly dependent" if @matrix_x.determinant<1e-15
+
+    
+    @matrix_y = @matrix_cor.submatrix(@fields, [y_var])
+    @matrix_y_cov = @matrix_cov.submatrix(@fields, [y_var])
+    
+    @y_sd=Math::sqrt(@matrix_cov.submatrix([y_var])[0,0])
+    
+    @x_sd=@n_predictors.times.inject({}) {|ac,i|
+      ac[@matrix_x_cov.fields[i]]=Math::sqrt(@matrix_x_cov[i,i])
+      ac;
+    }
+    
+    @cases=nil
+    @x_mean=@fields.inject({}) {|ac,f|
+      ac[f]=0.0
+      ac;
+    }
+    
+    @y_mean=0.0
+    @name=_("Multiple reggresion of %s on %s") % [@fields.join(","), @y_var]
+    
+    opts_default = {:digits=>3}
+    opts         = opts_default.merge opts
+    opts.each{|k,v|
+        self.send("#{k}=",v) if self.respond_to? k
+    }
+      result_matrix=@matrix_x_cov.inverse * @matrix_y_cov
+
+    if matrix._type == :covariance
+      @coeffs=result_matrix.column(0).to_a
+      @coeffs_stan=coeffs.collect {|k,v|
+        coeffs[k]*@x_sd[k].quo(@y_sd)
+      }
+    else
+      @coeffs_stan=result_matrix.column(0).to_a
+      @coeffs=standarized_coeffs.collect {|k,v|
+        standarized_coeffs[k]*@y_sd.quo(@x_sd[k])
+      } 
+    end
+    @total_cases=@valid_cases=@cases
+  end
+  def cases
+    raise "You should define the number of valid cases first" if @cases.nil?
+    @cases
+  end
+  # Get R^2 for the regression
+  # For fixed models is the coefficient of determination.
+  # On random models, is the 'squared-multiple correlation'
+  # Equal to 
+  # * 1-(|R| / |R_x|) or
+  # * Sum(b_i*r_yi) <- used
+  def r2
+    @n_predictors.times.inject(0) {|ac,i| ac+@coeffs_stan[i]* @matrix_y[i,0]} 
+  end
+  # Multiple correlation, on random models.
+  def r
+    Math::sqrt(r2)
+  end
+  # Value of constant
+  def constant
+    c = coeffs
+    @y_mean - @fields.inject(0) { |a,k| a + (c[k] * @x_mean[k])}
+  end
+  # Hash of b or raw coefficients
+  def coeffs
+    assign_names(@coeffs)
+  end
+  # Hash of beta or standarized coefficients
+
+  def standarized_coeffs
+    assign_names(@coeffs_stan)
+  end
+  # Total sum of squares
+  def sst
+    @y_sd**2*(cases-1.0)
+  end
+  
+  # Degrees of freedom for regression
+  def df_r
+    @n_predictors
+  end
+  # Degrees of freedom for error
+  def df_e
+    cases-@n_predictors-1
+  end
+  # Tolerance for a given variable
+  # defined as (1-R^2) of regression of other independent variables
+  # over the selected
+  # == Reference:
+  # * http://talkstats.com/showthread.php?t=5056
+  def tolerance(var)
+    return 1 if @matrix_x.column_size==1
+    lr=Statsample::Regression::Multiple::MatrixEngine.new(@matrix_x, var)
+    1-lr.r2
+  end
+  # Standard Error for coefficients.
+  # Standard error of a coefficients depends on
+  # * Tolerance of the coeffients: Higher tolerances implies higher error
+  # * Higher r2 implies lower error
+  # == Reference:
+  # * Cohen et al. (2003). Applied Multiple Reggression / Correlation Analysis for the Behavioral Sciences
+  #
+  def coeffs_se
+    out={}
+    #mse=sse.quo(df_e)
+    coeffs.each {|k,v|
+      out[k]=@y_sd.quo(@x_sd[k])*Math::sqrt( 1.quo(tolerance(k)))*Math::sqrt((1-r2).quo(df_e))
+    }
+    out
+  end
+  # t value for constant
+  def constant_t
+    return nil if constant_se.nil?
+    constant.to_f / constant_se
+  end
+  # Standard error for constant.
+  # This method recreates the estimaded variance-covariance matrix
+  # using means, standard deviation and covariance matrix.
+  # So, needs the covariance matrix.
+  def constant_se
+    return nil if @no_covariance
+    means=@x_mean
+    #means[@y_var]=@y_mean
+    means[:constant]=1
+    sd=@x_sd
+    #sd[@y_var]=@y_sd
+    sd[:constant]=0
+    fields=[:constant]+@matrix_cov.fields-[@y_var]
+    # Recreate X'X using the variance-covariance matrix
+    xt_x=::Matrix.rows(fields.collect {|i|
+      fields.collect {|j|
+        if i==:constant or j==:constant
+          cov=0
+        elsif i==j
+          cov=sd[i]**2
+        else
+          cov=@matrix_cov.submatrix(i..i,j..j)[0,0]
+        end
+        cov*(@cases-1)+@cases*means[i]*means[j]
+      }
+    })
+    matrix=xt_x.inverse * mse
+    matrix.collect {|i| Math::sqrt(i) if i>0 }[0,0]
+  end
+  
+end
+end
+end
+end

data/lib/statsample-ekatena/regression/multiple/rubyengine.rb

@@ -0,0 +1,86 @@
+module Statsample
+module Regression
+module Multiple
+# Pure Ruby Class for Multiple Regression Analysis.
+# Slower than AlglibEngine, but is pure ruby and can use a pairwise aproach for missing values. 
+# Coeffient calculation uses correlation matrix between the vectors
+# If you need listwise aproach for missing values, use AlglibEngine, because is faster.
+# 
+# Example:
+#
+#   @a = Daru::Vector.new([1,3,2,4,3,5,4,6,5,7])
+#   @b = Daru::Vector.new([3,3,4,4,5,5,6,6,4,4])
+#   @c = Daru::Vector.new([11,22,30,40,50,65,78,79,99,100])
+#   @y = Daru::Vector.new([3,4,5,6,7,8,9,10,20,30])
+#   ds = Daru::DataFrame.new({:a => @a,:b => @b,:c => @c,:y => @y})
+#   lr=Statsample::Regression::Multiple::RubyEngine.new(ds,:y)
+
+class RubyEngine < MatrixEngine
+  def initialize(ds,y_var, opts=Hash.new)
+    matrix = Statsample::Bivariate.correlation_matrix ds
+    fields_indep=ds.vectors.to_a - [y_var]
+    default= {
+      :y_mean => ds[y_var].mean,
+      :x_mean => fields_indep.inject({}) {|ac,f|  ac[f]=ds[f].mean; ac},
+      :y_sd   => ds[y_var].sd,
+      :x_sd   => fields_indep.inject({}) {|ac,f|  ac[f]=ds[f].sd; ac},
+      :cases  => Statsample::Bivariate.min_n_valid(ds)
+    }
+    opts = opts.merge(default)
+    super(matrix, y_var, opts)
+    @ds = ds
+    @dy = ds[@y_var]
+    @ds_valid = ds.reject_values(*Daru::MISSING_VALUES)
+    @total_cases = @ds.nrows
+    @valid_cases = @ds_valid.nrows
+    @ds_indep    = ds.dup(ds.vectors.to_a - [y_var])
+    set_dep_columns
+  end
+  
+  def set_dep_columns
+    @dep_columns = []
+    @ds_indep.each_vector { |v| @dep_columns.push(v.to_a) }                
+  end
+
+  def fix_with_mean
+    i=0
+    @ds_indep.each(:row) do |row|
+      empty=[]
+      row.each do |k,v|
+        empty.push(k) if v.nil?
+      end
+
+      if empty.size==1
+        @ds_indep[empty[0]][i]=@ds[empty[0]].mean
+      end
+      i += 1
+    end
+    set_dep_columns
+  end
+  def fix_with_regression
+    i = 0
+    @ds_indep.each(:row) do |row|
+      empty = []
+      row.each { |k,v| empty.push(k) if v.nil? }
+      if empty.size==1
+        field  = empty[0]
+        lr     = MultipleRegression.new(@ds_indep,field)
+        fields = []
+        @ds_indep.vectors.each { |f|
+          fields.push(row[f]) unless f == field
+        }
+        
+        @ds_indep[field][i]=lr.process(fields)
+      end
+      i+=1
+    end
+    set_dep_columns
+  end
+  # Standard error for constant
+  def constant_se
+    estimated_variance_covariance_matrix[0,0]
+  end
+end
+end
+end
+end

data/lib/statsample-ekatena/regression/simple.rb

@@ -0,0 +1,121 @@
+module Statsample
+  module Regression
+    # Class for calculation of linear regressions with form
+    #   y = a+bx
+    # To create a Statsample::Regression::Simple object:
+    # * <tt> Statsample::Regression::Simple.new_from_dataset(ds,x,y)</tt>
+    # * <tt> Statsample::Regression::Simple.new_from_vectors(vx,vy)</tt>
+    # * <tt> Statsample::Regression::Simple.new_from_gsl(gsl) </tt>
+    #
+    class Simple
+      include Summarizable
+      attr_accessor :a,:b,:cov00, :cov01, :covx1, :chisq, :status
+      attr_accessor :name
+      attr_accessor :digits
+      def initialize(init_method, *argv)
+        self.send(init_method, *argv)
+      end
+      private_class_method :new
+      # Obtain y value given x value
+      # x=a+bx
+
+      def y(val_x)
+        @a+@b*val_x
+      end
+      # Obtain x value given y value
+      # x=(y-a)/b
+      def x(val_y)
+        (val_y-@a) / @b.to_f
+      end
+      # Sum of square error
+      def sse
+        (0...@vx.size).inject(0) {|acum,i| acum+((@vy[i]-y(@vx[i]))**2)
+        }
+      end
+      def standard_error
+        Math::sqrt(sse / (@vx.size-2).to_f)
+      end
+      # Sum of square regression
+      def ssr
+        vy_mean=@vy.mean
+        (0...@vx.size).inject(0) {|a,i|
+          a+((y(@vx[i])-vy_mean)**2)
+        }
+
+      end
+      # Sum of square total
+      def sst
+        @vy.sum_of_squared_deviation
+      end
+      # Value of r
+      def r
+        @b * (@vx.sds / @vy.sds)
+      end
+      # Value of r^2
+      def r2
+        r**2
+      end
+      class << self
+        # Create a regression object giving an array with following parameters:
+        # <tt>a,b,cov00, cov01, covx1, chisq, status</tt>
+        # Useful to obtain x and y values with a and b values.
+        def new_from_gsl(ar)
+          new(:init_gsl, *ar)
+        end
+        # Create a simple regression using two vectors
+        def new_from_vectors(vx,vy, opts=Hash.new)
+          new(:init_vectors,vx,vy, opts)
+        end
+        # Create a simple regression using a dataset and two vector names.
+        def new_from_dataset(ds,x,y, opts=Hash.new)
+          new(:init_vectors,ds[x],ds[y], opts)
+        end
+      end
+      def init_vectors(vx,vy, opts=Hash.new)
+        @vx,@vy=Statsample.only_valid_clone(vx,vy)
+        x_m=@vx.mean
+        y_m=@vy.mean
+        num=den=0
+        (0...@vx.size).each {|i|
+          num+=(@vx[i]-x_m)*(@vy[i]-y_m)
+          den+=(@vx[i]-x_m)**2
+        }
+        @b=num.to_f/den
+        @a=y_m - @b*x_m
+        
+        opts_default={
+        :digits=>3, 
+        :name=>_("Regression of %s over %s") % [@vx.name, @vy.name]
+         }
+        @opts=opts_default.merge opts
+
+        @opts.each{|k,v|
+          self.send("#{k}=",v) if self.respond_to? k
+        }
+        
+      end
+      def init_gsl(a,b,cov00, cov01, covx1, chisq, status)
+        @a=a
+        @b=b
+        @cov00=cov00
+        @cov01=cov01
+        @covx1=covx1
+        @chisq=chisq
+        @status=status
+      end
+      def report_building(gen)
+      f="%0.#{digits}f"
+        gen.section(:name=>name) do |s|
+          s.table(:header=>[_("Variable"), _("Value")]) do |t|
+            t.row [_("r"), f % r]
+            t.row [_("r^2"), f % r2]
+            t.row [_("a"), f % a]
+            t.row [_("b"), f % b]
+            t.row [_("s.e"), f % standard_error]
+          end
+        end
+      end
+      private :init_vectors, :init_gsl
+    end
+  end
+end

data/lib/statsample-ekatena/reliability.rb

@@ -0,0 +1,150 @@
+module Statsample
+  module Reliability
+    class << self
+      # Calculate Chonbach's alpha for a given dataset.
+      # only uses tuples without missing data
+      def cronbach_alpha(ods)
+        ds = ods.reject_values(*Daru::MISSING_VALUES)
+        n_items = ds.ncols
+        return nil if n_items <= 1
+        s2_items = ds.to_hash.values.inject(0) { |ac,v| 
+          ac + v.variance }
+        total    = ds.vector_sum
+        
+        (n_items.quo(n_items - 1)) * (1 - (s2_items.quo(total.variance)))
+      end
+      # Calculate Chonbach's alpha for a given dataset
+      # using standarized values for every vector.
+      # Only uses tuples without missing data
+      # Return nil if one or more vectors has 0 variance
+      def cronbach_alpha_standarized(ods)
+        ds = ods.reject_values(*Daru::MISSING_VALUES)
+        return nil if ds.any? { |v| v.variance==0}
+        
+        ds = Daru::DataFrame.new(
+          ds.vectors.to_a.inject({}) { |a,i|
+            a[i] = ods[i].standardize
+            a
+          }
+        )
+                
+        cronbach_alpha(ds)
+      end
+      # Predicted reliability of a test by replicating
+      # +n+ times the number of items 
+      def spearman_brown_prophecy(r,n)
+        (n*r).quo(1+(n-1)*r)
+      end
+      
+      alias :sbp :spearman_brown_prophecy
+      # Returns the number of items 
+      # to obtain +r_d+ desired reliability
+      # from +r+ current reliability, achieved with
+      # +n+ items
+      def n_for_desired_reliability(r,r_d,n=1)
+        return nil if r.nil?
+        (r_d*(1-r)).quo(r*(1-r_d))*n
+      end
+      
+      # Get Cronbach alpha from <tt>n</tt> cases, 
+      # <tt>s2</tt> mean variance and <tt>cov</tt>
+      # mean covariance
+      def cronbach_alpha_from_n_s2_cov(n,s2,cov)
+        (n.quo(n-1)) * (1-(s2.quo(s2+(n-1)*cov)))
+      end
+      # Get Cronbach's alpha from a covariance matrix
+      def cronbach_alpha_from_covariance_matrix(cov)
+        n = cov.row_size
+        raise "covariance matrix should have at least 2 variables" if n < 2
+        s2 = n.times.inject(0) { |ac,i| ac + cov[i,i] }
+        (n.quo(n - 1)) * (1 - (s2.quo(cov.total_sum)))
+      end
+      # Returns n necessary to obtain specific alpha
+      # given variance and covariance mean of items
+      def n_for_desired_alpha(alpha,s2,cov)
+        # Start with a regular test : 50 items
+        min=2
+        max=1000
+        n=50
+        prev_n=0
+        epsilon=0.0001
+        dif=1000
+        c_a=cronbach_alpha_from_n_s2_cov(n,s2,cov)
+        dif=c_a - alpha
+        while(dif.abs>epsilon and n!=prev_n)
+          prev_n=n
+          if dif<0
+            min=n
+            n=(n+(max-min).quo(2)).to_i
+          else
+            max=n
+            n=(n-(max-min).quo(2)).to_i
+          end
+          c_a=cronbach_alpha_from_n_s2_cov(n,s2,cov)
+          dif=c_a - alpha
+        end
+        n
+      end
+      # First derivative for alfa
+      # Parameters
+      # <tt>n</tt>: Number of items
+      # <tt>sx</tt>: mean of variances 
+      # <tt>sxy</tt>: mean of covariances
+      
+      def alpha_first_derivative(n,sx,sxy)
+        (sxy*(sx-sxy)).quo(((sxy*(n-1))+sx)**2)
+      end
+      # Second derivative for alfa
+      # Parameters
+      # <tt>n</tt>: Number of items
+      # <tt>sx</tt>: mean of variances 
+      # <tt>sxy</tt>: mean of covariances
+      
+      def alfa_second_derivative(n,sx,sxy)
+        (2*(sxy**2)*(sxy-sx)).quo(((sxy*(n-1))+sx)**3)
+      end
+    end
+    class ItemCharacteristicCurve
+      attr_reader :totals, :counts, :vector_total
+      def initialize (ds, vector_total=nil)
+        vector_total||=ds.vector_sum
+        raise ArgumentError, "Total size != Dataset size" if vector_total.size != ds.nrows
+        @vector_total=vector_total
+        @ds=ds
+        @totals={}
+        @counts=@ds.vectors.to_a.inject({}) {|a,v| a[v]={};a}
+        process
+      end
+      def process
+        i=0
+        @ds.each_row do |row|
+          tot=@vector_total[i]
+          @totals[tot]||=0
+          @totals[tot]+=1
+          @ds.vectors.each  do |f|
+            item=row[f].to_s
+            @counts[f][tot]||={}
+            @counts[f][tot][item]||=0
+            @counts[f][tot][item] += 1
+          end
+          i+=1
+        end
+      end
+      # Return a hash with p for each different value on a vector
+      def curve_field(field, item)
+        out={}
+        item=item.to_s
+        @totals.each do |value,n|
+          count_value= @counts[field][value][item].nil? ? 0 : @counts[field][value][item]
+          out[value]=count_value.quo(n)
+        end
+        out
+      end # def
+    end # self
+  end # Reliability
+end # Statsample
+
+require 'statsample/reliability/icc.rb'
+require 'statsample/reliability/scaleanalysis.rb'
+require 'statsample/reliability/skillscaleanalysis.rb'
+require 'statsample/reliability/multiscaleanalysis.rb'