statsample 0.3.0

data/demo/stratum.rb

@@ -0,0 +1,141 @@
+require File.dirname(__FILE__)+"/../lib/statsample"
+require 'statsample/multiset'
+require 'statsample/srs'
+require 'statsample/resample'
+require 'gnuplot'
+
+tests=3000
+sample_size=50
+
+a=[10]*50+[12]*10+[14]*20+[16]*10+[19]*10
+b=[11000]*50+[11050]*10+[11100]*20+[11300]*10+[11240]*10
+a_size=a.size
+b_size=b.size
+av=a.to_vector(:scale)
+bv=b.to_vector(:scale)
+
+ads={'data'=>a.to_vector(:scale)}.to_dataset
+bds={'data'=>b.to_vector(:scale)}.to_dataset
+
+m=Statsample::Multiset.new(['data'])
+m.add_dataset('a',ads)
+m.add_dataset('b',bds)
+ss=Statsample::StratifiedSample.new(m,{'a'=>a.size,'b'=>b.size})
+
+es=[{'N'=>a_size,'n'=>sample_size/2,'s'=>av.standard_deviation_population}, {'N'=>b_size,'n'=>sample_size/2,'s'=>bv.standard_deviation_population}]
+
+
+
+sd_estimated_wr=Statsample::StratifiedSample.standard_error_ksd_wr(es)
+
+sd_estimated_wor = Statsample::StratifiedSample.standard_error_ksd_wor(es)
+
+
+
+pop=(a+b).to_vector(:scale)
+s=pop.standard_deviation_population
+
+
+
+
+puts "-------------"
+
+puts "Estadísticos:"
+puts "Mean:"+pop.mean.to_s
+puts "SD:"+s.to_s
+puts "EE con reemplazo:"+Statsample::SRS.standard_error_ksd_wr(s, sample_size, pop.size).to_s
+puts "EE sin reemplazo:"+Statsample::SRS.standard_error_ksd_wor(s, sample_size,pop.size).to_s
+
+puts "EE estratified con reemplazo:"+sd_estimated_wr.to_s
+puts "EE estratified sin reemplazo:"+sd_estimated_wor.to_s
+sd_with=[]
+sd_without=[]
+sd_strat_wr=[]
+sd_strat_wor=[]
+monte_with=Statsample::Resample.repeat_and_save(tests) {
+    sample= pop.sample_with_replacement(sample_size)
+    sd_with.push(Statsample::SRS.standard_error_esd_wr(sample.sds,sample_size,pop.size))
+    sample.mean
+}
+
+
+monte_without=Statsample::Resample.repeat_and_save(tests) {
+    sample= pop.sample_without_replacement(sample_size)
+    sd_without.push(Statsample::SRS.standard_error_esd_wor(sample.sds,sample_size,pop.size))
+    sample.mean
+}
+
+
+
+stratum_wor=Statsample::Resample.repeat_and_save(tests) {
+    a_sample= {'data'=>av.sample_without_replacement(sample_size/2)}.to_dataset
+    b_sample= {'data'=>bv.sample_without_replacement(sample_size/2)}.to_dataset
+    m=Statsample::Multiset.new(['data'])
+    m.add_dataset('a',a_sample)
+    m.add_dataset('b',b_sample)
+    ss=Statsample::StratifiedSample.new(m,{'a'=>a_size,'b'=>b_size})
+    sd_strat_wor.push(ss.standard_error_wor('data'))
+    ss.mean('data')    
+}.to_vector(:scale)
+
+stratum_wr=Statsample::Resample.repeat_and_save(tests) {
+    a_sample= {'data'=>av.sample_with_replacement(sample_size/2)}.to_dataset
+    b_sample= {'data'=>bv.sample_with_replacement(sample_size/2)}.to_dataset
+    m=Statsample::Multiset.new(['data'])
+    m.add_dataset('a',a_sample)
+    m.add_dataset('b',b_sample)
+    ss=Statsample::StratifiedSample.new(m,{'a'=>a_size,'b'=>b_size})
+    sd_strat_wr.push(ss.standard_error_wr('data'))
+    ss.mean('data')    
+}.to_vector(:scale)
+
+
+
+v_sd_with=sd_with.to_vector(:scale)
+v_sd_without=sd_without.to_vector(:scale)
+v_sd_strat_wr=sd_strat_wr.to_vector(:scale)
+v_sd_strat_wor=sd_strat_wor.to_vector(:scale)
+
+
+v_with=monte_with.to_vector(:scale)
+v_without=monte_without.to_vector(:scale)
+puts "=============="
+puts "Con reemplazo"
+puts "Mean:"+v_with.mean.to_s
+puts "Sd:"+v_with.sds.to_s
+puts "Sd (estimated):"+v_sd_with.mean.to_s
+puts "=============="
+puts "Sin reemplazo"
+puts "Mean:"+v_without.mean.to_s
+puts "Sd:"+v_without.sds.to_s
+puts "Sd (estimated):"+v_sd_without.mean.to_s
+puts "=============="
+puts "Estratificado Con reemplazo"
+puts "Mean:"+stratum_wr.mean.to_s
+puts "Sd:"+stratum_wr.sds.to_s
+puts "Sd (estimated):"+v_sd_strat_wr.mean.to_s
+
+puts "=============="
+puts "Estratificado Sin reemplazo"
+puts "Mean:"+stratum_wor.mean.to_s
+puts "Sd:"+stratum_wor.sds.to_s
+puts "Sd (estimated):"+v_sd_strat_wor.mean.to_s
+
+p v_without.plot_histogram
+
+=begin
+
+
+
+x=[]
+y=[]
+y2=[]
+prev=0
+prev_chi=0
+v.frequencies.sort.each{|k,v1|
+	x.push(k)
+	y.push(prev+v1)
+	prev=prev+v1
+}
+GSL::graph(GSL::Vector.alloc(x),  GSL::Vector.alloc(y))
+=end

data/lib/spss.rb

@@ -0,0 +1,131 @@
+# = spss.rb - 
+#
+# Provides utilites for working with spss files
+#
+# Copyright (C) 2009 Claudio Bustos
+#
+# Claudio Bustos mailto:clbustos@gmail.com
+
+module SPSS
+    module Dictionary
+        class Element
+            def add(a)
+                @elements.push(a)
+            end
+            def parse_elements(func=:to_s)
+                @elements.collect{|e| "   "+e.send(func)}.join("\n")
+            end
+            def init_with config
+                config.each {|key,value|
+                    self.send(key.to_s+"=",value) if methods.include? key.to_s
+                }
+            end
+            def initialize(config={})
+                @config=config
+                @elements=[]
+            end
+        end
+        class Dictionary < Element
+            attr_accessor :locale, :date_time, :row_count
+            def initialize(config={})
+                super
+                init_with ({
+                        :locale=>"en_US", 
+                        :date_time=>Time.new().strftime("%Y-%m-%dT%H:%M:%S"),
+                        :row_count=>1
+                })
+                init_with config
+            end
+            
+            def to_xml
+                "<dictionary locale='#{@locale}' creationDateTime='#{@date_time}' rowCount='#{@row_count}' xmlns='http://xml.spss.com/spss/data'>\n"+parse_elements(:to_xml)+"\n</dictionary>"
+                
+            end
+            def to_spss
+                parse_elements(:to_spss)
+            end
+        end
+        
+        class MissingValue < Element
+            attr_accessor :data, :type, :from, :to
+            def initialize(data,type=nil)
+                @data=data
+                if type.nil? or type=="lowerBound" or type=="upperBound"
+                    @type=type
+                else
+                    raise Exception,"Incorrect value for type"
+                end
+            end
+            def to_xml
+                "<missingValue data='#{@data}' "+(type.nil? ? "":"type='#{type}'")+"/>"
+            end
+        end
+        class LabelSet
+            attr_accessor
+            def initialize(labels)
+                @labels=labels
+            end
+            def parse_xml(name)
+                "<valueLabelSet>\n   "+@labels.collect{|key,value| "<valueLabel label='#{key}' value='#{value}' />"}.join("\n   ")+"\n   <valueLabelVariable name='#{name}' />\n</valueLabelSet>"
+            end
+            def parse_spss()
+                @labels.collect{|key,value| "#{key} '#{value}'"}.join("\n   ")
+            end
+        end
+        class Variable < Element
+            attr_accessor :aligment, :display_width, :label, :measurement_level, :name, :type, :decimals, :width, :type_format, :labelset, :missing_values
+            def initialize(config={})
+                super
+                @@var_number||=1
+                init_with({
+                        :aligment           =>  "left",
+                        :display_width      =>  8,
+                        :label              =>  "Variable #{@@var_number}",
+                        :measurement_level  =>  "SCALE",
+                        :name               =>  "var#{@@var_number}",
+                        :type               =>  0,
+                        :decimals           =>  2,
+                        :width              =>  10,
+                        :type_format        =>  "F",
+                        :labelset           => nil
+                })
+                init_with config
+                @missing_values=[]
+                @@var_number+=1
+            end
+            def to_xml
+                labelset_s=(@labelset.nil?) ? "":"\n"+@labelset.parse_xml(@name)
+                missing_values=(@missing_values.size>0) ? @missing_values.collect {|m| m.to_xml}.join("\n"):""
+                "<variable aligment='#{@aligment}' displayWidth='#{@display_width}' label='#{@label}' measurementLevel='#{@measurement_level}' name='#{@name}' type='#{@type}'>\n<variableFormat decimals='#{@decimals}' width='#{@width}' type='#{@type_format}' />\n"+parse_elements(:to_xml)+missing_values+"</variable>"+labelset_s
+            end
+            def to_spss
+                out=<<HERE
+VARIABLE LABELS #{@name} '#{label}' .
+VARIABLE ALIGMENT #{@name} (#{@aligment.upcase}) .
+VARIABLE WIDTH #{@name} (#{@display_width}) .
+VARIABLE LEVEL #{@name} (#{@measurement_level.upcase}) .
+HERE
+                if !@labelset.nil?
+                    out << "VALUE LABELS #{@name} "+labelset.parse_spss()+" ."
+                end
+                if @missing_values.size>0
+                    out << "MISSING VALUES #{@name} ("+@missing_values.collect{|m| m.data}.join(",")+") ."
+                end
+                out
+            end
+        end
+    end
+end
+n=SPSS::Dictionary::Dictionary.new
+ls=SPSS::Dictionary::LabelSet.new({1=>"Si",2=>"No"})
+var1=SPSS::Dictionary::Variable.new
+var1.labelset=ls
+mv1=SPSS::Dictionary::MissingValue.new("-99")
+var2=SPSS::Dictionary::Variable.new
+n.add(var1)
+n.add(var2)
+var2.missing_values=[mv1]
+
+File.open("dic_spss.sps","wb") {|f|
+    f.puts n.to_spss
+}

data/lib/statsample.rb

@@ -0,0 +1,216 @@
+# = statsample.rb - 
+
+# Process files and databases for statistical purposes, with focus on 
+# estimation of parameters for several types of samples (simple random, 
+# stratified and multistage sampling).
+#
+# Copyright (C) 2008-2009 Claudio Bustos
+#
+# Claudio Bustos mailto:clbustos_AT_gmail.com
+
+# :stopdoc:
+
+$:.unshift(File.dirname(__FILE__))
+$:.unshift(File.expand_path(File.dirname(__FILE__)+"/../ext"))
+
+require 'delegate'
+require 'matrix'
+
+
+class Numeric
+  def square ; self * self ; end
+end
+
+
+def create_test(*args,&proc) 
+    description=args.shift
+    fields=args
+    [description, fields, Proc.new]
+end
+
+# Process files and databases for statistical purposes, with focus on 
+# estimation of parameters for several types of samples (simple random, 
+# stratified and multistage sampling).
+
+begin
+	require 'rbgsl'
+	HAS_GSL=true
+rescue LoadError
+	HAS_GSL=false
+end
+    begin 
+        require 'alglib'
+        HAS_ALGIB=true
+    rescue LoadError
+        HAS_ALGIB=false
+    end
+        
+    
+    begin 
+        require 'statsample/optimization'
+    rescue LoadError
+        module Statsample
+            OPTIMIZED=false
+        end
+    end
+
+#
+# :startdoc:
+#
+module Statsample
+    VERSION = '0.3.0'
+    SPLIT_TOKEN = ","
+	autoload(:Database, 'statsample/converters')
+    autoload(:Anova, 'statsample/anova')
+	autoload(:CSV, 'statsample/converters')
+	autoload(:Excel, 'statsample/converters')
+	autoload(:GGobi, 'statsample/converters')
+    autoload(:DominanceAnalysis, 'statsample/dominanceanalysis')
+	autoload(:HtmlReport, 'statsample/htmlreport')
+    autoload(:Mx, 'statsample/converters')
+	autoload(:Resample, 'statsample/resample')
+	autoload(:SRS, 'statsample/srs')
+	autoload(:Codification, 'statsample/codification')
+	autoload(:Reliability, 'statsample/reliability')
+	autoload(:Bivariate, 'statsample/bivariate')
+	autoload(:Multivariate, 'statsample/multivariate')
+
+	autoload(:Regression, 'statsample/regression')
+	autoload(:Test, 'statsample/test')
+    def self.load(filename)
+        fp=File.open(filename,"r")
+        o=Marshal.load(fp)
+        fp.close
+        o
+    end
+    
+	module Util
+        # Reference: http://www.itl.nist.gov/div898/handbook/eda/section3/normprpl.htm
+        def normal_order_statistic_medians(i,n)
+            if i==1
+                u= 1.0 - normal_order_statistic_medians(n,n)
+            elsif i==n
+                u=0.5**(1 / n.to_f)
+            else
+                u= (i - 0.3175) / (n + 0.365)
+            end
+            u
+        end
+	end
+    module Writable
+        def save(filename)
+            fp=File.open(filename,"w")
+            Marshal.dump(self,fp)
+            fp.close
+        end        
+    end
+    module HtmlSummary
+        def add_line(n=nil)
+            self << "<hr />"
+        end
+        def nl
+            self << "<br />"
+        end
+        def add(text)
+            self << ("<p>"+text.gsub("\n","<br />")+"</p>")
+        end
+        def parse_table(table)
+            self << table.parse_html
+        end
+    end
+    module ConsoleSummary
+        def add_line(n=80)
+            self << "-"*n+"\n"
+        end
+        def nl
+            self << "\n"
+        end
+        def add(text)
+            self << text
+        end
+        def parse_table(table)
+            self << table.parse_console
+        end
+    end
+    class ReportTable
+        attr_reader :header
+        def initialize(header=[])
+            @header=header
+            @rows=[]
+            @max_cols=[]
+        end
+        def add_row(row)
+            row.each_index{|i|
+                @max_cols[i]=row[i].to_s.size if @max_cols[i].nil? or row[i].to_s.size > @max_cols[i]
+            }
+            @rows.push(row)
+        end
+        def add_horizontal_line
+            @rows.push(:hr)
+        end
+        def header=(h)
+            h.each_index{|i|
+                @max_cols[i]=h[i].to_s.size if @max_cols[i].nil? or h[i].to_s.size>@max_cols[i]
+            }    
+            @header=h
+        end
+        def parse_console_row(row)
+            out="| "
+            @max_cols.each_index{|i|
+                if row[i].nil?
+                    out << " "*(@max_cols[i]+2)+"|"
+                else
+                    t=row[i].to_s
+                    out << " "+t+" "*(@max_cols[i]-t.size+1)+"|"
+                end
+            }
+            out << "\n"
+            out
+        end
+        def parse_console_hr
+            "-"*(@max_cols.inject(0){|a,v|a+v.size+3}+2)+"\n"
+        end
+        def parse_console
+            out="\n"
+            out << parse_console_hr
+            out << parse_console_row(header)
+            out << parse_console_hr
+
+            @rows.each{|row|
+                if row==:hr
+                   out << parse_console_hr 
+                else
+                out << parse_console_row(row)
+                end
+            }
+            out << parse_console_hr
+
+            out
+        end
+        def parse_html
+            out="<table>\n"
+            if header.size>0
+            out << "<thead><th>"+header.join("</th><th>")+"</thead><tbody>"
+            end
+            out << "<tbody>\n"
+            row_with_line=false
+            @rows.each{|row|
+                if row==:hr
+                    row_with_line=true
+                else
+                    out << "<tr class='"+(row_with_line ? 'line':'')+"'><td>"
+                    out << row.join("</td><td>") +"</td>"
+                    out << "</tr>\n"
+                    row_with_line=false
+                end
+            }
+            out << "</tbody></table>\n"
+            out
+        end
+    end
+end
+
+require 'statsample/vector'
+require 'statsample/dataset'
+require 'statsample/crosstab'
+