statsample 0.16.0 → 0.17.0

data/lib/statsample/graph/histogram.rb

@@ -0,0 +1,133 @@
+require 'rubyvis'
+module Statsample
+  module Graph
+    class Histogram
+      include Summarizable
+      attr_accessor :name
+      # Total width of Boxplot
+      attr_accessor :width
+      # Total height of Boxplot
+      attr_accessor :height
+      # Top margin
+      attr_accessor :margin_top
+      # Bottom margin
+      attr_accessor :margin_bottom
+      # Left margin
+      attr_accessor :margin_left
+      # Right margin
+      attr_accessor :margin_right
+      attr_reader :hist
+      # Could be an array of ranges or number of bins
+      attr_accessor :bins
+      # data could be a vector or a histogram
+      def initialize(data,opts=Hash.new)
+        prov_name=data.respond_to? :name ? data.name : ""
+        opts_default={
+          :name=>_("Histograma (%s)") % prov_name,
+          :width=>400,
+          :height=>300,
+          :margin_top=>10,
+          :margin_bottom=>20,
+          :margin_left=>20,
+          :margin_right=>20,
+          :bins=>nil
+        }
+        @opts=opts_default.merge(opts)
+        opts_default.keys.each {|k| send("#{k}=", @opts[k]) }
+        @data=data
+      end
+      def pre_vis
+        if @data.is_a? Statsample::Histogram
+          @hist=@data
+        elsif @data.is_a? Statsample::Vector
+          @bins||=Math::sqrt(@data.size).floor
+          @hist=@data.histogram(@bins)
+        end
+      end
+      # Returns a Rubyvis panel with scatterplot
+      def rubyvis_panel # :nodoc:
+        pre_vis
+        that=self
+        
+        max_bin = @hist.max_val
+        
+        margin_hor=margin_left + margin_right
+        margin_vert=margin_top  + margin_bottom
+      
+        x_scale = pv.Scale.linear(@hist.min, @hist.max).range(0,width-margin_hor)
+      
+        y_scale=Rubyvis::Scale.linear(0,max_bin).range(0, height-margin_vert)
+        
+        y_scale.nice
+        max_range=@hist.max
+        bins=@hist.bins.times.map {|i|
+          {
+           :low =>@hist.get_range(i)[0],
+           :high=>@hist.get_range(i)[1],
+           :value=>@hist.bin[i]
+          }
+        }
+        # cache data
+        vis=Rubyvis::Panel.new do |pan| 
+          pan.width  width  - margin_hor
+          pan.height height - margin_vert
+          pan.bottom margin_bottom
+          pan.left   margin_left
+          pan.right  margin_right
+          pan.top    margin_top
+           # Y axis
+          pan.rule do
+            data y_scale.ticks
+            bottom y_scale
+            stroke_style {|d| d!=0 ? "#eee" : "#000"}
+            label(:anchor=>'left') do
+              text y_scale.tick_format
+            end
+          end
+          # X axis
+          pan.rule do
+            data x_scale.ticks
+            left x_scale
+            stroke_style "black"
+            height 5
+            bottom -5
+            label(:anchor=>'bottom') do
+              text x_scale.tick_format
+            end
+          end
+          
+          pan.bar do |bar|
+            bar.data(bins)
+            bar.left {|v| x_scale.scale(v[:low])}
+            bar.width {|v| x_scale.scale(v[:high]) - x_scale.scale(v[:low])}
+            bar.bottom 0
+            bar.height {|v| y_scale.scale(v[:value])}
+            bar.stroke_style "black"
+            bar.line_width 1
+          end
+        end
+      end
+      # Returns SVG with scatterplot
+      def to_svg
+        rp=rubyvis_panel
+        rp.render
+        rp.to_svg
+      end
+      def report_building(builder) # :nodoc:
+        builder.section(:name=>name) do |b|
+          b.image(to_svg, :type=>'svg', :width=>width, :height=>height)
+        end
+      end
+      def report_building_text(generator)
+        pre_vis
+        #anchor=generator.toc_entry(_("Histogram %s") % [@name])
+        step=  @hist.max_val > 40 ? ( @hist.max_val / 40).ceil : 1
+          
+        @hist.range.each_with_index do |r,i|
+          next if i==@hist.bins
+          generator.text(sprintf("%5.2f : %s", r, "*" * (@hist.bin[i] / step).floor ))
+        end
+      end
+    end
+  end
+end

data/lib/statsample/graph/scatterplot.rb

@@ -1,4 +1,3 @@
-require 'tmpdir'
 require 'rubyvis'
 module Statsample
   module Graph
@@ -152,15 +151,8 @@ module Statsample
         rp.to_svg
       end
       def report_building(builder) # :nodoc:
-        img_svg=to_svg
         builder.section(:name=>name) do |b|
-          Dir.mktmpdir {|dir|
-            time=Time.new.to_f
-            File.open("#{dir}/image_#{time}.svg","w") {|fp|
-              fp.write img_svg
-            }
-            b.image("#{dir}/image_#{time}.svg", :width=>width, :height=>height)
-          }
+          b.image(to_svg, :type=>'svg', :width=>width, :height=>height)
         end
         
       end

data/lib/statsample/histogram.rb

@@ -39,10 +39,24 @@ module Statsample
   
     class Histogram
       class << self
+        # Alloc +n_bins+, using +range+ as ranges of bins
         def alloc(n_bins, range=nil, opts=Hash.new)
-          Histogram.new(n_bins, range)
+          Histogram.new(n_bins, range, opts)
           
         end
+        # Alloc +n_bins+ bins, using +p1+ as minimum and +p2+
+        # as maximum
+        def alloc_uniform(n_bins, p1=nil,p2=nil)
+          if p1.is_a? Array
+            min,max=p1
+          else
+            min,max=p1,p2
+          end
+          range=max - min
+          step=range / n_bins.to_f
+          range=(n_bins+1).times.map {|i| min + (step*i)}
+          Histogram.new(range)
+        end
       end
       attr_accessor :name
       attr_reader :bin
@@ -53,25 +67,29 @@ module Statsample
         
         if p1.is_a? Array
           range=p1
-          n_bins=p1.size-1
+          @n_bins=p1.size-1
         elsif p1.is_a? Integer
-          n_bins=p1
+          @n_bins=p1
         end
         
-        @bin=[0.0]*(n_bins)
+        @bin=[0.0]*(@n_bins)
         if(min_max)
           min, max=min_max[0], min_max[1]
-          range=Array.new(n_bins+1)
-          (n_bins+1).times {|i| range[i]=min+(i*(max-min).quo(n_bins)) }
+          range=Array.new(@n_bins+1)
+          (@n_bins+1).times {|i| range[i]=min+(i*(max-min).quo(@n_bins)) }
         end
-        range||=[0.0]*(n_bins+1)
+        range||=[0.0]*(@n_bins+1)
         set_ranges(range)
         @name=""
         opts.each{|k,v|
         self.send("#{k}=",v) if self.respond_to? k
         }
       end
-      
+      # Number of bins
+      def bins
+        @n_bins
+      end
+      # 
       def increment(x, w=1)
         if x.is_a? Array
           x.each{|y| increment(y,w) }
@@ -88,11 +106,29 @@ module Statsample
         raise "Range size should be bin+1" if range.size!=@bin.size+1
         @range=range
       end
+      def get_range(i)
+        [@range[i],@range[i+1]]
+      end
+      def max
+        @range.last
+      end
+      def min
+        @range.first
+      end
+      def max_val
+        @bin.max
+      end
+      def min_val
+        @bin.min
+      end
+      def report_building(generator)
+        hg=Statsample::Graph::Histogram.new(self)
+        generator.parse_element(hg)
+      end
       def report_building_text(generator)
-        anchor=generator.toc_entry(_("Histogram %s") % [@name])
-        range.each_with_index do |r,i|
+        @range.each_with_index do |r,i|
           next if i==@bin.size
-          generator.text(sprintf("%4.2f : %d", r, @bin[i]))
+          generator.text(sprintf("%5.2f : %d", r, @bin[i]))
         end
       end
     end

data/lib/statsample/mle.rb

@@ -41,7 +41,7 @@ module Statsample
         x.row_size.times{|i|
           xi=Matrix.rows([x.row(i).to_a.collect{|v| v.to_f}])
           y_val=y[i,0].to_f
-          fbx=f(b,x)
+          #fbx=f(b,x)
           prod=prod*likehood_i(xi, y_val ,b)
         }
         prod
@@ -62,7 +62,7 @@ module Statsample
       def set_default_parameters(x)
         fd=[0.0]*x.column_size
         fd.push(0.1)    if self.is_a? Statsample::MLE::Normal
-        parameters = Matrix.columns([fd])
+        Matrix.columns([fd])
       end
       
       # Newton Raphson with automatic stopping criteria.
@@ -80,8 +80,8 @@ module Statsample
             parameters = start_values.dup
         end
         k=parameters.row_size
-        cv=Matrix.rows([([1.0]*k)])
-        last_diff=nil
+        #cv=Matrix.rows([([1.0]*k)])
+        #last_diff=nil
         raise "n on y != n on x" if x.row_size!=y.row_size
         h=nil
         fd=nil

data/lib/statsample/mle/normal.rb

@@ -39,7 +39,7 @@ module Statsample
                 rows = Array.new(k+1)
                 k.times{|i| rows[i] = [xte[i,0] / sigma2]}
                 rows[k] = [ete[0,0] / (2*sigma4) - n / (2*sigma2)]
-                fd = Matrix.rows(rows, true)
+                Matrix.rows(rows, true)
             end
             
             # second derivative for normal model
@@ -48,7 +48,7 @@ module Statsample
                 raise "x.rows!=y.rows" if x.row_size!=y.row_size
                 raise "x.columns+1!=p.rows" if x.column_size+1!=p.row_size
 
-                n = x.row_size
+                #n = x.row_size
                 k = x.column_size
                 b = Array.new(k)
                 k.times{|i| b[i]=[p[i,0]]}
@@ -76,7 +76,7 @@ module Statsample
                 end
                 last_row[k] = 2*sigma4 - ete[0,0] / sigma6
                 rows[k] = last_row
-                sd = Matrix.rows(rows, true)
+                Matrix.rows(rows, true)
             end
         end
     end

data/lib/statsample/regression/multiple/baseengine.rb

@@ -150,7 +150,7 @@ module Statsample
         # Estimated Variance-Covariance Matrix
         # Used for calculation of se of constant
         def estimated_variance_covariance_matrix
-          mse_p=mse
+          #mse_p=mse
           columns=[]
           @ds_valid.fields.each{|k|
             v=@ds_valid[k]

data/lib/statsample/regression/multiple/gslengine.rb

@@ -66,7 +66,6 @@ class GslEngine < BaseEngine
   # Coefficients using a constant
   # Based on http://www.xycoon.com/ols1.htm
   def matrix_resolution
-    mse_p=mse
     columns=@dep_columns.dup.map {|xi| xi.map{|i| i.to_f}}
     columns.unshift([1.0]*@ds.cases)
     y=Matrix.columns([@dy.data.map  {|i| i.to_f}])

data/lib/statsample/regression/multiple/matrixengine.rb

@@ -157,7 +157,7 @@ class MatrixEngine < BaseEngine
   #
   def coeffs_se
     out={}
-    mse=sse.quo(df_e)
+    #mse=sse.quo(df_e)
     coeffs.each {|k,v|
       out[k]=@y_sd.quo(@x_sd[k])*Math::sqrt( 1.quo(tolerance(k)))*Math::sqrt((1-r2).quo(df_e))
     }

data/lib/statsample/reliability.rb

@@ -141,4 +141,5 @@ end # Statsample
 
 require 'statsample/reliability/icc.rb'
 require 'statsample/reliability/scaleanalysis.rb'
+require 'statsample/reliability/skillscaleanalysis.rb'
 require 'statsample/reliability/multiscaleanalysis.rb'

data/lib/statsample/reliability/scaleanalysis.rb

@@ -40,7 +40,7 @@ module Statsample
         @sd = @total.sd
         @variance=@total.variance
         @valid_n = @total.size
-        opts_default={:name=>"Reliability Analisis"}
+        opts_default={:name=>_("Reliability Analisis")}
         @opts=opts_default.merge(opts)
         @name=@opts[:name]
         @cov_m=Statsample::Bivariate.covariance_matrix(@ds)
@@ -79,56 +79,6 @@ module Statsample
         end
         out
       end
-      def gnuplot_item_characteristic_curve(directory, base="crd",options={})
-        require 'gnuplot'
-
-        crd=item_characteristic_curve
-        @ds.fields.each  do |f|
-          x=[]
-          y=[]
-          Gnuplot.open do |gp|
-            Gnuplot::Plot.new( gp ) do |plot|
-              crd[f].sort.each do |tot,prop|
-                x.push(tot)
-                y.push((prop*100).to_i.to_f/100)
-              end
-              plot.data << Gnuplot::DataSet.new( [x, y] ) do |ds|
-                ds.with = "linespoints"
-                ds.notitle
-              end
-
-            end
-          end
-        end
-      end
-      def svggraph_item_characteristic_curve(directory, base="icc",options={})
-        require 'statsample/graph/svggraph'
-        crd=ItemCharacteristicCurve.new(@ds)
-        @ds.fields.each do |f|
-          factors=@ds[f].factors.sort
-          options={
-            :height=>500,
-            :width=>800,
-            :key=>true
-          }.update(options)
-          graph = ::SVG::Graph::Plot.new(options)
-          factors.each do |factor|
-            factor=factor.to_s
-            dataset=[]
-            crd.curve_field(f, factor).each do |tot,prop|
-              dataset.push(tot)
-              dataset.push((prop*100).to_i.to_f/100)
-            end
-            graph.add_data({
-              :title=>"#{factor}",
-              :data=>dataset
-            })
-          end
-          File.open(directory+"/"+base+"_#{f}.svg","w") {|fp|
-            fp.puts(graph.burn())
-          }
-        end
-      end
       # =Adjusted R.P.B. for each item
       # Adjusted RPB(Point biserial-correlation) for each item
       #
@@ -172,9 +122,11 @@ module Statsample
         ds_new.update_valid_data
         ds_new
       end
+      
       def stats_if_deleted
         @sif||=stats_if_deleted_intern
       end
+      
       def stats_if_deleted_intern # :nodoc:
         return Hash.new if @ds.fields.size==1
         @ds.fields.inject({}) do |a,v|

data/lib/statsample/reliability/skillscaleanalysis.rb

@@ -0,0 +1,93 @@
+module Statsample
+  module Reliability
+    # Analysis of a Skill Scale
+    # Given a dataset with results and a correct answers hash,
+    # generates a ScaleAnalysis 
+    # == Usage
+    #  x1=%{a b b c}.to_vector
+    #  x2=%{b a b c}.to_vector
+    #  x3=%{a c b a}.to_vector
+    #  ds={'x1'=>@x1,'x2'=>@x2,'x3'=>@x3}.to_dataset
+    #  key={'x1'=>'a','x2'=>'b','x3'=>'a'}    
+    #  ssa=Statsample::Reliability::SkillScaleAnalysis.new(ds,key)
+    #  puts ssa.summary
+    class SkillScaleAnalysis
+      include Summarizable
+      attr_accessor :name
+      attr_accessor :summary_minimal_item_correlation
+      attr_accessor :summary_show_problematic_items
+      def initialize(ds,key,opts=Hash.new)
+        opts_default={
+          :name=>_("Skill Scale Reliability Analysis"),
+          :summary_minimal_item_correlation=>0.10,
+          :summary_show_problematic_items=>true
+        }
+        @ds=ds
+        @key=key
+        @opts=opts_default.merge(opts)
+        @opts.each{|k,v| self.send("#{k}=",v) if self.respond_to? k }
+        @cds=nil
+      end
+      def corrected_dataset_minimal
+        cds=corrected_dataset
+        @key.keys.inject({}) {|ac,v| ac[v]=cds[v];ac}.to_dataset
+      end
+      def vector_sum
+        corrected_dataset_minimal.vector_sum
+      end
+      def vector_mean
+        corrected_dataset_minimal.vector_mean
+      end
+      def scale_analysis
+        sa=ScaleAnalysis.new(corrected_dataset_minimal)
+        sa.name=_("%s (Scale Analysis)") % @name
+        sa
+      end
+      def corrected_dataset
+        if @cds.nil?
+          @cds=@ds.dup_empty
+          @key.keys.each {|k| @cds[k].type=:scale; @cds[k].name=@ds[k].name}
+          @ds.each do |row|
+            out={}
+            row.each do |k,v|
+              if @key.keys.include? k
+                if @ds[k].is_valid? v
+                  out[k]= @key[k]==v ? 1 : 0
+                else
+                  out[k]=nil
+                end
+              else
+                out[k]=v
+              end
+            end
+            @cds.add_case(out,false)
+          end
+          @cds.update_valid_data
+        end
+        @cds
+      end
+      def report_building(builder)
+        builder.section(:name=>@name) do |s|
+          sa=scale_analysis
+          s.parse_element(sa)
+          if summary_show_problematic_items
+            s.section(:name=>_("Problematic Items")) do |spi|
+              count=0
+              sa.item_total_correlation.each do |k,v|
+                if v<summary_minimal_item_correlation
+                  count+=1
+                  spi.section(:name=>_("Item: %s") % @ds[k].name) do |spii|
+                    spii.text _("Correct answer: %s") % @key[k]
+                    spii.parse_element(@ds[k])
+                  end
+                end
+              end
+              spi.text _("No problematic items") if count==0
+            end
+          end
+          
+        end
+      end
+    end    
+  end
+end