statsample-ekatena 2.0.2

data/lib/statsample-ekatena/reliability/scaleanalysis.rb

@@ -0,0 +1,233 @@
+module Statsample
+  module Reliability
+    # Analysis of a Scale. Analoge of Scale Reliability analysis on SPSS.
+    # Returns several statistics for complete scale and each item
+    # == Usage
+    #  @x1 = Daru::Vector.new([1,1,1,1,2,2,2,2,3,3,3,30])
+    #  @x2 = Daru::Vector.new([1,1,1,2,2,3,3,3,3,4,4,50])
+    #  @x3 = Daru::Vector.new([2,2,1,1,1,2,2,2,3,4,5,40])
+    #  @x4 = Daru::Vector.new([1,2,3,4,4,4,4,3,4,4,5,30])
+    #  ds  = Daru::DataFrame.new({:x1 => @x1,:x2 => @x2,:x3 => @x3,:x4 => @x4})
+    #  ia  = Statsample::Reliability::ScaleAnalysis.new(ds)
+    #  puts ia.summary
+    class ScaleAnalysis
+      include Summarizable
+      attr_reader :ds,:mean, :sd,:valid_n, :alpha , :alpha_standarized, :variances_mean, :covariances_mean, :cov_m
+      attr_accessor :name
+      attr_accessor :summary_histogram
+      def initialize(ds, opts=Hash.new)
+        @dumped=ds.vectors.to_a.find_all {|f|
+          ds[f].variance == 0
+        }
+        
+        @ods = ds
+        @ds  = ds.reject_values(*Daru::MISSING_VALUES).dup(ds.vectors.to_a - @dumped)
+        @ds.rename ds.name
+        
+        @k     = @ds.ncols
+        @total = @ds.vector_sum
+        @o_total=@dumped.size > 0 ? @ods.vector_sum : nil
+        
+        @vector_mean = @ds.vector_mean
+        @item_mean   = @vector_mean.mean
+        @item_sd     = @vector_mean.sd
+        
+        @mean     = @total.mean
+        @median   = @total.median
+        @skew     = @total.skew
+        @kurtosis = @total.kurtosis
+        @sd       = @total.sd
+        @variance = @total.variance
+        @valid_n  = @total.size
+
+        opts_default = {
+          :name => _("Reliability Analysis"),
+          :summary_histogram => true
+        }
+        @opts = opts_default.merge(opts)
+        @opts.each{ |k,v| self.send("#{k}=",v) if self.respond_to? k }
+        
+        @cov_m=Statsample::Bivariate.covariance_matrix(@ds)
+        # Mean for covariances and variances
+        @variances = Daru::Vector.new(@k.times.map { |i| @cov_m[i,i] })
+        @variances_mean=@variances.mean
+        @covariances_mean=(@variance-@variances.sum).quo(@k**2-@k)
+        #begin
+          @alpha = Statsample::Reliability.cronbach_alpha(@ds)
+          @alpha_standarized = Statsample::Reliability.cronbach_alpha_standarized(@ds)
+        #rescue => e
+        #  raise DatasetException.new(@ds,e), "Error calculating alpha"
+        #end
+      end
+      # Returns a hash with structure
+      def item_characteristic_curve
+        i=0
+        out={}
+        total={}
+        @ds.each do |row|
+          tot=@total[i]
+          @ds.vectors.each do |f|
+            out[f]||= {}
+            total[f]||={}
+            out[f][tot]||= 0
+            total[f][tot]||=0
+            out[f][tot]+= row[f]
+            total[f][tot]+=1
+          end
+          i+=1
+        end
+        total.each do |f,var|
+          var.each do |tot,v|
+            out[f][tot]=out[f][tot].quo(total[f][tot])
+          end
+        end
+        out
+      end
+      # =Adjusted R.P.B. for each item
+      # Adjusted RPB(Point biserial-correlation) for each item
+      #
+      def item_total_correlation
+        vecs = @ds.vectors.to_a
+        @itc ||= vecs.inject({}) do |a,v|
+          total=@ds.vector_sum(vecs - [v])
+          a[v]=Statsample::Bivariate.pearson(@ds[v],total)
+          a
+        end
+      end
+      def mean_rpb
+        Daru::Vector.new(item_total_correlation.values).mean
+      end
+      def item_statistics
+        @is||=@ds.vectors.to_a.inject({}) do |a,v|
+          a[v]={:mean=>@ds[v].mean, :sds=>Math::sqrt(@cov_m.variance(v))}
+          a
+        end
+      end
+      # Returns a dataset with cases ordered by score
+      # and variables ordered by difficulty
+
+      def item_difficulty_analysis
+        dif={}
+        @ds.vectors.each{|f| dif[f]=@ds[f].mean }
+        dif_sort = dif.sort { |a,b| -(a[1]<=>b[1]) }
+        scores_sort={}
+        scores=@ds.vector_mean
+        scores.each_index{ |i| scores_sort[i]=scores[i] }
+        scores_sort=scores_sort.sort{|a,b| a[1]<=>b[1]}
+        ds_new = Daru::DataFrame.new({}, order: ([:case,:score] + dif_sort.collect{|a,b| a.to_sym}))
+        scores_sort.each do |i,score|
+          row = [i, score]
+          case_row = @ds.row[i].to_hash
+          dif_sort.each{ |variable,dif_value| row.push(case_row[variable]) }
+          ds_new.add_row(row)
+        end
+        ds_new
+      end
+      
+      def stats_if_deleted
+        @sif||=stats_if_deleted_intern
+      end
+      
+      def stats_if_deleted_intern # :nodoc:
+        return Hash.new if @ds.ncols == 1
+        vecs = @ds.vectors.to_a
+        vecs.inject({}) do |a,v|
+          cov_2=@cov_m.submatrix(vecs - [v])
+          #ds2=@ds.clone
+          #ds2.delete_vector(v)
+          #total=ds2.vector_sum
+          a[v]={}
+          #a[v][:mean]=total.mean
+          a[v][:mean]=@mean-item_statistics[v][:mean]
+          a[v][:variance_sample]=cov_2.total_sum
+          a[v][:sds]=Math::sqrt(a[v][:variance_sample])
+          n=cov_2.row_size
+          a[v][:alpha] = (n>=2) ? Statsample::Reliability.cronbach_alpha_from_covariance_matrix(cov_2) : nil
+          a
+        end
+      end
+      def report_building(builder) #:nodoc:
+        builder.section(:name=>@name) do |s|
+          
+          if @dumped.size>0
+            s.section(:name=>"Items with variance=0") do |s1|
+              s.table(:name=>_("Summary for %s with all items") % @name) do |t|
+                t.row [_("Items"), @ods.ncols]
+                t.row [_("Sum mean"),     "%0.4f" % @o_total.mean]
+                t.row [_("S.d. mean"),     "%0.4f" % @o_total.sd]
+              end
+              s.table(:name=>_("Deleted items"), :header=>['item','mean']) do |t|
+                @dumped.each do |f|
+                  t.row(["#{@ods[f].name}(#{f})", "%0.5f" % @ods[f].mean])
+                end
+              end
+              s.parse_element(Statsample::Graph::Histogram.new(@o_total, :name=>"Histogram (complete data) for %s" % @name)) if @summary_histogram
+            end
+          end
+          
+          
+          s.table(:name=>_("Summary for %s") % @name) do |t|
+            t.row [_("Valid Items"), @ds.ncols]
+          
+          t.row [_("Valid cases"), @valid_n]
+          t.row [_("Sum mean"),     "%0.4f" % @mean]
+          t.row [_("Sum sd"),       "%0.4f" % @sd  ]
+#          t.row [_("Sum variance"), "%0.4f" % @variance]
+          t.row [_("Sum median"),   @median]
+          t.hr
+          t.row [_("Item mean"),    "%0.4f" % @item_mean]
+          t.row [_("Item sd"),    "%0.4f" % @item_sd]
+          t.hr
+          t.row [_("Skewness"),     "%0.4f" % @skew]
+          t.row [_("Kurtosis"),     "%0.4f" % @kurtosis]
+          t.hr
+          t.row [_("Cronbach's alpha"), @alpha ? ("%0.4f" % @alpha) : "--"]
+          t.row [_("Standarized Cronbach's alpha"), @alpha_standarized ? ("%0.4f" % @alpha_standarized) : "--" ]
+          t.row [_("Mean rpb"), "%0.4f" % mean_rpb]
+          
+          t.row [_("Variances mean"),  "%g" % @variances_mean]
+          t.row [_("Covariances mean") , "%g" % @covariances_mean]
+          end
+          
+          if (@alpha)
+            s.text _("Items for obtain alpha(0.8) : %d" % Statsample::Reliability::n_for_desired_reliability(@alpha, 0.8, @ds.ncols))
+            s.text _("Items for obtain alpha(0.9) : %d" % Statsample::Reliability::n_for_desired_reliability(@alpha, 0.9, @ds.ncols))
+          end
+          
+          
+          sid=stats_if_deleted
+          is=item_statistics
+          itc=item_total_correlation
+          
+          s.table(:name=>_("Items report for %s") % @name, :header=>["item","mean","sd", "mean if deleted", "var if deleted", "sd if deleted"," item-total correl.", "alpha if deleted"]) do |t|
+            @ds.vectors.each do |f|
+              row=["#{@ds[f].name}(#{f})"]
+              if is[f]
+                row+=[sprintf("%0.5f",is[f][:mean]), sprintf("%0.5f", is[f][:sds])]
+              else
+                row+=["-","-"]
+              end
+              if sid[f]
+                row+= [sprintf("%0.5f",sid[f][:mean]), sprintf("%0.5f",sid[f][:variance_sample]), sprintf("%0.5f",sid[f][:sds])]
+              else
+                row+=%w{- - -}
+              end
+              if itc[f]
+                row+= [sprintf("%0.5f",itc[f])]
+              else 
+                row+=['-']
+              end
+              if sid[f] and !sid[f][:alpha].nil?
+                row+=[sprintf("%0.5f",sid[f][:alpha])]
+              else
+                row+=["-"]
+              end
+              t.row row
+            end # end each
+          end # table
+          s.parse_element(Statsample::Graph::Histogram.new(@total, :name=>"Histogram (valid data) for %s" % @name)) if @summary_histogram
+        end # section
+      end # def
+    end # class
+  end # module
+end # module

data/lib/statsample-ekatena/reliability/skillscaleanalysis.rb

@@ -0,0 +1,114 @@
+module Statsample
+  module Reliability
+    # Analysis of a Skill Scale
+    # Given a dataset with results and a correct answers hash,
+    # generates a ScaleAnalysis 
+    # == Usage
+    #  x1 = Daru::Vector.new(%{a b b c})
+    #  x2 = Daru::Vector.new(%{b a b c})
+    #  x3 = Daru::Vector.new(%{a c b a})
+    #  ds = Daru::DataFrame.new({:x1 => @x1, :x2 => @x2, :x3 => @x3})
+    #  key={ :x1 => 'a',:x2 => 'b', :x3 => 'a'}    
+    #  ssa=Statsample::Reliability::SkillScaleAnalysis.new(ds,key)
+    #  puts ssa.summary
+    class SkillScaleAnalysis
+      include Summarizable
+      attr_accessor :name
+      attr_accessor :summary_minimal_item_correlation
+      attr_accessor :summary_show_problematic_items
+      def initialize(ds,key,opts=Hash.new)
+        opts_default={
+          :name=>_("Skill Scale Reliability Analysis (%s)") % ds.name,
+          :summary_minimal_item_correlation=>0.10,
+          :summary_show_problematic_items=>true
+        }
+        @ds=ds
+        @key=key
+        @opts=opts_default.merge(opts)
+        @opts.each{|k,v| self.send("#{k}=",v) if self.respond_to? k }
+        @cds=nil
+      end
+      # Dataset only corrected vectors
+      def corrected_dataset_minimal
+        cds = corrected_dataset
+        dsm = Daru::DataFrame.new(
+          @key.keys.inject({}) do |ac,v| 
+            ac[v] = cds[v]
+            ac
+          end
+        )
+        
+        dsm.rename _("Corrected dataset from %s") % @ds.name
+        dsm
+      end
+
+      def vector_sum
+        corrected_dataset_minimal.vector_sum
+      end
+
+      def vector_mean
+        corrected_dataset_minimal.vector_mean
+      end
+
+      def scale_analysis
+        sa = ScaleAnalysis.new(corrected_dataset_minimal)
+        sa.name=_("%s (Scale Analysis)") % @name
+        sa
+      end
+
+      def corrected_dataset
+        if @cds.nil?
+          @cds = Daru::DataFrame.new({}, order: @ds.vectors, name: @ds.name)
+          @ds.each_row do |row|
+            out = {}
+            row.each_with_index do |v, k|
+              if @key.has_key? k
+                if @ds[k].reject_values(*Daru::MISSING_VALUES).include_values? v
+                  out[k]= @key[k] == v ? 1 : 0
+                else
+                  out[k] = nil
+                end
+              else
+                out[k] = v
+              end
+            end
+
+            @cds.add_row(Daru::Vector.new(out))
+          end
+          @cds.update
+        end
+        @cds
+      end
+
+      def report_building(builder)
+        builder.section(:name=>@name) do |s|
+          sa = scale_analysis
+          s.parse_element(sa)
+          if summary_show_problematic_items
+            s.section(:name=>_("Problematic Items")) do |spi|
+              count=0
+              sa.item_total_correlation.each do |k,v|
+                if v < summary_minimal_item_correlation
+                  count+=1
+                  spi.section(:name=>_("Item: %s") % @ds[k].name) do |spii|
+                    spii.text _("Correct answer: %s") % @key[k]
+                    spii.text _("p: %0.3f") % corrected_dataset[k].mean
+                    props=@ds[k].proportions.inject({}) {|ac,v| ac[v[0]] = v[1].to_f;ac}
+                    
+                    spi.table(:name=>"Proportions",:header=>[_("Value"), _("%")]) do |table|
+                      props.each do |k1,v|
+                        table.row [ @ds[k].index_of(k1), "%0.3f" % v]
+                      end
+                    end
+                  end
+                end
+              end
+
+              spi.text _("No problematic items") if count==0
+            end
+          end
+        end
+      end
+    end    
+  end
+end

data/lib/statsample-ekatena/resample.rb

@@ -0,0 +1,15 @@
+module Statsample
+  module Resample
+    class << self
+      def repeat_and_save(times,&action)
+        (1..times).inject([]) {|a,x| a.push(action.call); a}
+      end
+      
+      def generate (size,low,upper)
+        range=upper-low+1
+        Daru::Vector.new((0...size).collect {|x| rand(range)+low })
+      end
+              
+    end
+  end
+end

data/lib/statsample-ekatena/shorthand.rb

@@ -0,0 +1,125 @@
+class Object
+  # Shorthand for Statsample::Analysis.store(*args,&block)
+  def ss_analysis(*args,&block)
+    Statsample::Analysis.store(*args,&block)
+  end
+end
+
+module Statsample
+  # Module which provide shorthands for many methods.
+  module Shorthand
+    ###
+    # :section: R like methods
+    ###
+
+    # Import an Excel file. Cache result by default
+    def read_excel(filename, opts=Hash.new)
+      Daru::DataFrame.from_excel filename, opts
+    end
+
+    # Import an CSV file. Cache result by default
+    def read_csv(filename, opts=Hash.new)
+      Daru::DataFrame.from_csv filename, opts
+    end
+    
+    # Retrieve names (fields) from dataset
+    def names(ds)
+      ds.vectors.to_a
+    end
+    # Create a correlation matrix from a dataset
+    def cor(ds)
+      Statsample::Bivariate.correlation_matrix(ds)
+    end
+    # Create a variance/covariance matrix from a dataset
+    def cov(ds)
+      Statsample::Bivariate.covariate_matrix(ds)
+    end
+    # Create a Daru::Vector
+    # Analog to R's c
+    def vector(*args)
+      Daru::Vector[*args]
+    end
+    # Random generation for the normal distribution
+    def rnorm(n,mean=0,sd=1)
+      rng=Distribution::Normal.rng(mean,sd)
+      Daru::Vector.new_with_size(n) { rng.call}
+    end
+    # Creates a new Daru::DataFrame
+    # Each key is transformed into a Symbol wherever possible.
+    def dataset(vectors=Hash.new)
+      vectors = vectors.inject({}) do |ac,v| 
+        n     = v[0].respond_to?(:to_sym) ? v[0].to_sym : v[0] 
+        ac[n] = v[1]
+        ac
+      end
+      Daru::DataFrame.new(vectors)
+    end
+    alias :data_frame :dataset
+    # Returns a Statsample::Graph::Boxplot
+    def boxplot(*args)
+      Statsample::Graph::Boxplot.new(*args)
+    end
+    # Returns a Statsample::Graph::Histogram
+    def histogram(*args)
+      Statsample::Graph::Histogram.new(*args)
+    end
+    
+    # Returns a Statsample::Graph::Scatterplot
+    def scatterplot(*args)
+      Statsample::Graph::Scatterplot.new(*args)
+    end
+    # Returns a Statsample::Test::Levene
+    def levene(*args)
+      Statsample::Test::Levene.new(*args)
+    end
+
+    def principal_axis(*args)
+      Statsample::Factor::PrincipalAxis.new(*args)
+    end
+
+    def polychoric(*args)
+      Statsample::Bivariate::Polychoric.new(*args)
+    end
+
+    def tetrachoric(*args)
+      Statsample::Bivariate::Tetrachoric.new(*args)
+    end
+
+    ###
+    # Other Shortcuts
+    ###
+    def lr(*args)
+      Statsample::Regression.multiple(*args)
+    end
+
+    def pca(ds,opts=Hash.new)
+      Statsample::Factor::PCA.new(ds,opts)
+    end
+
+    def dominance_analysis(*args)
+      Statsample::DominanceAnalysis.new(*args)
+    end
+
+    def dominance_analysis_bootstrap(*args)
+      Statsample::DominanceAnalysis::Bootstrap.new(*args)
+    end
+
+    def scale_analysis(*args)
+      Statsample::Reliability::ScaleAnalysis.new(*args)
+    end
+
+    def skill_scale_analysis(*args)
+      Statsample::Reliability::SkillScaleAnalysis.new(*args)
+    end
+
+    def multiscale_analysis(*args,&block)
+      Statsample::Reliability::MultiScaleAnalysis.new(*args,&block)
+    end
+
+    def test_u(*args)
+      Statsample::Test::UMannWhitney.new(*args)
+    end
+    
+    module_function :test_u, :rnorm
+  end
+end