statsample-sem 0.1.0

data/History.txt

@@ -0,0 +1,3 @@
+=== 0.1.0 / 2010-07-03
+
+* First operational version, with 'sem' and 'OpenMx' R's libraries.

data/Manifest.txt

@@ -0,0 +1,18 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/statsample_sem
+example/normal_vs_saturated.rb
+lib/statsample/sem.rb
+lib/statsample/sem/model.rb
+lib/statsample/sem/openmxengine.rb
+lib/statsample/sem/semjfoxengine.rb
+spec/fixtures/demo_open_mx.csv
+spec/fixtures/demo_open_mx.ds
+spec/spec.opts
+spec/spec_helper.rb
+spec/statsample_sem_model_spec.rb
+spec/statsample_sem_openmxengine_spec.rb
+spec/statsample_sem_semjfoxengine_spec.rb
+spec/statsample_sem_spec.rb

data/README.txt

@@ -0,0 +1,56 @@
+= statsample-sem
+
+* http://ruby-statsample.rubyforge.org/
+
+== DESCRIPTION:
+
+Structural equation modeling (SEM) for statsample gem, usign ruby and R
+
+== FEATURES/PROBLEMS:
+
+* Abstract generation of models. You could use R packages sem or OpenMx only changing the configuration for engine
+* Generates visual representations of output models using GraphViz (to implement)
+
+== SYNOPSIS:
+
+    require 'statsample'
+    require 'statsample/sem'
+    require 'matrix'    
+    matrix=Matrix[[0.1985443,0.1999953,0.2311884,0.2783865,0.3155943],
+    [0.1999953,0.2916950,0.2924566,0.3515298,0.4019234],
+    [0.2311884,0.2924566,0.3740354,0.4061291,0.4573587],
+    [0.2783865,0.3515298,0.4061291,0.5332788,0.5610769],
+    [0.3155943,0.4019234,0.4573587,0.5610769,0.6703023]]
+    
+    cases=500
+    sem1=Statsample::SEM.new do |m|
+        m.data_from_matrix(matrix,:cases=>cases)
+        m.variables=%w{x1 x2 x3 x4 x5} # Variables on matrix
+        m.manifests %w{x1 x2 x3 x4 x5}
+        m.latents %w{G}
+        m.path :from=>m.latents, :to=>m.manifests
+        m.path :from=>m.manifests
+        m.path :from=>m.latents, :free=>false, :values=>1.0
+    end
+    sem1.compute
+    puts sem1.chi_square
+    
+    sem2=sem1.dup
+    sem2.make_null # Generate a null model
+    sem2.compute
+    puts sem2.chi_square
+    puts sem2.df
+
+
+== REQUIREMENTS:
+
+* statsample
+* R with sem and/or OpenMx packages installed
+
+== INSTALL:
+
+* sudo gem install statsample-sem
+
+== LICENSE:
+
+GPL-2 

data/Rakefile

@@ -0,0 +1,16 @@
+# -*- ruby -*-
+$:.unshift(File.dirname(__FILE__)+"/lib")
+require 'rubygems'
+require 'hoe'
+require 'statsample/sem'
+Hoe.plugin :git
+
+
+Hoe.spec 'statsample-sem' do
+  self.version=Statsample::SEM::VERSION
+  self.rubyforge_name = 'ruby-statsample'
+  self.developer('Claudio Bustos', 'clbustos_at_gmail.com')
+  self.extra_deps << ["statsample","~>0.13.1"] << ["rserve-client", "~>0.2.0"] << ['dirty-memoize']
+end
+
+# vim: syntax=ruby

data/bin/statsample_sem

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+abort "you need to write me"

data/example/normal_vs_saturated.rb

@@ -0,0 +1,36 @@
+$:.unshift(File.dirname(__FILE__)+"/../lib/")
+require 'statsample'
+require 'statsample/sem'
+require 'matrix'
+
+
+matrix=Matrix[[0.1985443,0.1999953,0.2311884,0.2783865,0.3155943],
+[0.1999953,0.2916950,0.2924566,0.3515298,0.4019234],
+[0.2311884,0.2924566,0.3740354,0.4061291,0.4573587],
+[0.2783865,0.3515298,0.4061291,0.5332788,0.5610769],
+[0.3155943,0.4019234,0.4573587,0.5610769,0.6703023]]
+
+
+cases=500
+sem1=Statsample::SEM.new do |m|
+    m.data_from_matrix(matrix,:cases=>cases)
+    m.variables=%w{x1 x2 x3 x4 x5}
+    m.manifests %w{x1 x2 x3 x4 x5}
+    m.latents %w{G}
+    m.path :from=>m.latents, :to=>m.manifests
+    m.path :from=>m.manifests
+    m.path :from=>m.latents, :free=>false, :values=>1.0
+    
+end
+sem1.compute
+puts sem1.chi_square
+
+sem2=sem1.dup
+sem2.make_null
+sem2.compute
+
+
+
+puts sem2.chi_square
+puts sem2.df
+

data/lib/statsample/sem.rb

@@ -0,0 +1,46 @@
+require 'rserve'
+require 'dirty-memoize'
+require 'forwardable'
+require 'statsample'
+require 'statsample/rserve_extension'
+require 'statsample/sem/model'
+require 'statsample/sem/openmxengine'
+require 'statsample/sem/semjfoxengine'
+
+module Statsample
+  class SEM
+    extend Forwardable
+    VERSION='0.1.0'
+    attr_accessor :name
+    attr_accessor :engine
+    def_delegators :@model, :path, :data_from_matrix, :data_from_dataset, :manifests, :manifests=, :latents, :latents=, :variables=, :make_null
+      def_delegators :@engine_obj, :chi_square, :df
+    def initialize(opts=Hash.new, &block)
+      default_opts={:name=>"SEM Analysis", :engine=>:sem}
+      @opts=default_opts.merge(opts)
+      @engine_obj=nil
+      @name=@opts.delete :name
+      @engine=@opts.delete :engine
+      @model=Statsample::SEM::Model.new(:name=>@name)
+      if block
+        block.arity<1 ? instance_eval(&block) : block.call(self)
+      end
+    end
+    def compute
+      @engine_obj=case @engine
+      when :openmx
+        OpenMxEngine.new(@model)
+      when :sem
+        SemJFoxEngine.new(@model)
+      end
+      @engine_obj.compute
+      @engine_obj
+    end
+    def engine_obj
+      @engine_obj||=compute
+    end
+    def r_summary
+      engine_obj.r_summary
+    end
+  end
+end

data/lib/statsample/sem/model.rb

@@ -0,0 +1,244 @@
+module Statsample
+  class SEM
+    class Model
+      include Summarizable
+      # Type of data used. Could be +:covariance+, +:correlation+ or +:raw+
+      attr_reader :data_type
+      # Covariance/correlation matrix
+      attr_reader :matrix
+      # Raw data on a dataset
+      attr_reader :ds
+      # Number of cases
+      attr_accessor :cases
+      # Optional array of mean for use when data is matrix based
+      attr_accessor :means
+      # Name of variables
+      attr_reader :variables
+      
+      attr_reader :paths
+      def initialize(opts=Hash.new, &block)
+        raise ArgumentError,"opts should be a Hash" if !opts.is_a? Hash
+        default_opts={:name=>_("SEM Model")}
+        @opts=default_opts.merge opts
+        @paths=Hash.new
+        @manifests=nil
+        @latents=nil
+        if block
+          block.arity<1 ? self.instance_eval(&block) : block.call(self)
+        end
+      end
+      def dup
+        mod=Model.new(@opts)
+        if @data_type==:raw
+          mod.data_from_dataset(@ds)
+        else
+          mod.data_from_matrix(@matrix,:type=>@data_type,:cases=>@cases, :means=>@means) 
+        end
+        mod.latents=latents
+        mod.manifests=manifests
+        if @paths
+          @paths.each_pair do |k,v|
+            mod.add_raw_path(k[0],k[1],v[:arrow],v[:label],v[:free], v[:value])
+          end
+        end
+        mod
+      end
+      def make_null
+        # Get current variables
+        vars=variables_on_paths
+        @paths.clear
+        vars.each do |v|
+          if @variables.include? v
+            free=true
+            value=nil
+          else
+            free=false
+            value=1.0
+          end
+          add_raw_path(v,v,2,"var #{v}",free,value)
+        end
+      end
+      # True if model have enough information to process it
+      def complete?
+        !@data_type.nil? and !@manifests.nil? and !@latents.nil? and @paths.size>0
+      end
+      def add_raw_path(f1,f2,arrows,label,free,value)
+        @paths[[f1,f2].sort]={ :from=>f1, :to=>f2, :arrow=>arrows, :label=>label, :free=>free, :value=>value}
+      end
+      def add_path(f1,f2,arrows,free,values,labels,i)
+        arrow_s = (arrows==1) ? "to":"cov"
+        raise "Path repeated : #{f1},#{f2}" if @paths.has_key? [[f1,f2].sort]
+        label= (labels.nil? or !labels.respond_to?:[] or labels[i].nil?) ? "#{f1} #{arrow_s} #{f2}" : labels[i]
+        
+        free_v = (free.is_a? Array) ? free[i] : free 
+        if values.is_a? Array
+          value= values[i].nil? ? nil : values[i]
+        elsif values.is_a? Numeric
+          value=values
+        end
+          
+        value = nil if free_v
+        
+        @paths[[f1,f2].sort]={ :from=>f1, :to=>f2, :arrow=>arrows, :label=>label, :free=>free_v, :value=>value}
+      end
+      # Set one or more paths. Based on OpenMx mxPath method.
+      # 
+      # ==Options:
+      # * +:from+   : String or Array. sources of new paths
+      # * +:to+     : String or Array. sinks of new paths
+      # * +:all+    : bool. If you, connect all sources to all sinks. If false,
+      # connect one-on-one sources to sinks if both are arrays, one source to
+      # many sinks if +:from+ is a String and +:to+ is an Array
+      # * +:arrows+ : 1 for regression, 2 for variance-covariance. See rules
+      # for specific automatic setting
+      # * +:free+   : Indicates whether paths are free or fixed. By default, true
+      # * +:values+ : The starting values of the parameters
+      # * +:labels+ : The names of paths
+      # ==Rules
+      # * from : variance-> from and to equal, label equal to "s^2 NAME_OF_FIELD", arrows=2, free=>true
+      # * from and to: regression -> label equal to "FROM->TO", arrows=1, free=>true
+      # * from, to, arrows -> label equal to "FROM->TO" if arrows=1,
+      # "FROM<->TO" if arrows=2, free=>true
+      # * free=false -> requires values for each from - to value
+      
+      def path(opts)
+        raise "Requires at least :from option" unless opts.has_key? :from
+        
+        free=true
+        all=false
+        
+        from=opts[:from]
+        to=opts[:to]
+        
+        all=opts[:all] if opts.has_key? :all
+        free=opts[:free] if opts.has_key? :free
+
+        labels=opts[:labels]
+        arrows=opts[:arrows]
+        
+        values=opts[:values]
+        from=[from] if from.is_a? String
+        to||=from
+        to=[to] if to.is_a? String
+        
+        if from==to # variances
+          arrows||=2
+          labels_2=Array.new(from.size)
+          
+          from.each_with_index do |f,i|
+            labels_2[i]=(labels.nil? or !labels.respond_to?:[] or labels[i].nil?) ? "var #{f}" : labels[i]
+          end
+          
+          from.each_with_index do |f,i|
+            add_path(f,f,arrows,free,values, labels_2,i)
+          end
+        else # regression and covariances
+          arrows||=1
+          i=0
+          all=true if from.size==1 or to.size==1
+          if all
+            from.each do |f1|
+              to.each do |f2|
+                add_path(f1,f2,arrows,free,values,labels,i)
+                i+=1 
+              end
+            end
+          else
+            raise ":from and :to should be the same size" if from.size!=to.size
+            from.size.times.each do |i|
+              add_path(from[i],to[i],arrows,free,values,labels,i)
+            end
+            
+          end
+          
+        end
+        
+        
+      end
+      def variables=(v)
+        raise ArgumentError, "Should be size=#{@variables.row_size}" if @data_type!=:raw and v.size!=@matrix.row_size
+        @variables=v.map {|i| i.to_s}
+      end
+      # Total number of variables
+      def variables_on_paths
+        out=Array.new
+        @paths.keys.each {|k|
+          out << k[0] unless out.include? k[0]
+          out << k[1] unless out.include? k[1]
+        }
+        out
+      end
+      # Number of variables on model
+      def k
+        variables_on_paths.size
+      end
+      # Latents will be any variable set on a path not present
+      # on @variables
+      def infer_latents
+        variables_on_paths-@variables
+      end
+      # Latens will be any variable set on path present on @variables
+      def infer_manifests
+        variables_on_paths & @variables
+      end
+      def get_label(key)
+        @paths[key][:label]
+      end
+      def latents(*argv)
+        if argv.size==0
+          if @latents.nil? and @paths.size>0 and !@data_type.nil? 
+            @latents=infer_latents
+          end
+          @latents
+        elsif argv[0].is_a? Array 
+          @latents=argv[0]
+        else
+          @latents=[argv]
+        end
+      end
+      def latents=(argv)
+        @latents=argv
+      end
+      
+      def manifests(*argv)
+        if argv.size==0
+          if @manifests.nil? and @paths.size>0 and !@data_type.nil? 
+            @manifests=infer_manifests
+          end
+          @manifests
+        elsif argv[0].is_a? Array 
+          @manifests=argv[0]
+        else
+          @manifests=[argv]
+        end
+      end
+      def manifests=(argv)
+        @manifests=argv
+      end
+      def data_from_dataset(ds)
+        @data_type=:raw
+        @ds=ds
+        @variables=@ds.fields
+      end
+      
+      def data_from_matrix(matrix,opts=Hash.new)
+        type = opts[:type]
+        type||=(matrix.respond_to? :type) ? matrix.type : :covariance
+        variable_names = opts[:variable_names]
+        cases = opts[:cases]
+        means = opts[:means]
+        raise "You should set number of cases" if cases.nil?
+        
+        @data_type= (type==:covariance) ? :covariance : :correlation
+        @matrix=matrix
+        @cases=cases
+        @means=means
+        if variable_names.nil? 
+          @variables=@matrix.fields if @matrix.respond_to? :fields
+        else
+          @variables=variable_names
+        end
+      end
+    end
+  end
+end

data/lib/statsample/sem/openmxengine.rb

@@ -0,0 +1,144 @@
+require 'rserve'
+require 'tempfile'
+module Statsample
+  class SEM
+    class OpenMxEngine
+      include DirtyMemoize
+      include Summarizable
+      attr_accessor :summarizable
+      attr_accessor :name
+      attr_reader :r_summary
+      def initialize(model,opts=Hash.new)
+        @model=model
+        defaults = {
+          :name=>_("SEM analysis using OpenMx")
+        }
+        @opts=defaults.merge defaults
+        @name=@opts[:name]
+      end
+      def r
+        @r||=Rserve::Connection.new
+      end
+      def r_mxpaths
+        @model.paths.values.map {|path|
+          value= path[:value] ? ", values = #{path[:value]}":""
+          label= path[:label] ? ", labels = \"#{path[:label]}\"" : ""  
+          "mxPath(from=c(\"#{path[:from]}\"), to=c(\"#{path[:to]}\"), arrows = #{path[:arrow]}, free = #{path[:free] ? "TRUE" : "FALSE"} #{value} #{label})"
+        }.join(",\n")
+      end
+      def r_mxdata
+        type=case @model.data_type
+          when :raw
+            'raw'
+          when :covariance
+            'cov'
+          when :correlation
+            'cor'
+        end
+        means=(@model.data_type!=:raw and !@model.means.nil?) ? ", means = d_means " : ""
+        num=(@model.data_type!=:raw) ? ", numObs = #{@model.cases} " : ""
+        
+        "mxData(observed=data, type='#{type}' #{means} #{num})"
+      end
+        
+      def r_query
+        <<-EOF
+library(OpenMx);
+factorModel <- mxModel(
+name="#{name}",
+type="RAM",
+manifestVars = manifests,
+latentVars = latents,
+#{r_mxpaths}, 
+#{r_mxdata}
+);
+factorFit<-mxRun(factorModel);
+rm(data,manifests,latents,d_means);
+        EOF
+        #p r.eval('factorFit').to_ruby
+      end
+      
+      def compute
+        raise "Insuficient information" unless @model.complete?
+        r.assign 'data', @model.data_type==:raw ? @model.ds : @model.matrix
+        if @model.matrix
+          r.assign 'vn', @model.variables
+          # We should assing names to fields on matrix
+          r.void_eval('dimnames(data)<-list(vn,vn)')
+        end
+        r.assign 'manifests',@model.manifests
+        r.assign 'latents', @model.latents
+        r.assign 'd_means',@model.means unless @model.means.nil?
+        r.void_eval r_query
+        @r_summary=@r.eval('summary(factorFit)').to_ruby
+        true
+      end
+      def graphviz
+        compute if @r_summary.nil?
+        tf=Tempfile.new('model.dot')
+        r.void_eval("omxGraphviz(factorModel,'#{tf.path}')")
+#        tf.close
+#        tf.open
+        tf.read
+      end
+      def chi_square
+        @r_summary['Chi']
+      end
+      def df
+        @r_summary['degreesOfFreedom']
+      end
+      def chi_square_null
+        null_model.r_summary['Chi']
+      end
+      def df_null
+        null_model.r_summary['degreesOfFreedom']
+      end
+
+      def null_model
+        @null_model||=compute_null_model
+      end
+      def compute_null_model #:nodoc:
+        nm=@model.dup
+        nm.make_null
+        (self.class).new(nm,@opts)
+      end
+      def rmsea
+        @r_summary['RMSEA']
+      end
+       # [χ2(Null Model) - χ2(Proposed Model)]/ [χ2(Null Model)]
+      def nfi
+        (chi_square_null-chi_square).quo(chi_square_null)
+      end
+      
+      # [χ2/df(Null Model) - χ2/df(Proposed Model)]/[χ2/df(Null Model) - 1]
+      def nnfi
+        (chi_square_null.quo(df_null) - chi_square.quo(df)).quo(chi_square_null.quo(df_null)-1)
+      end
+      def cfi
+        d_null=chi_square_null-df_null
+        ((d_null)-(chi_square-df)).quo(d_null)
+      end
+      def bic
+        k=@model.k
+        p k
+        ln_n=Math.log(@model.cases)
+        chi_square+((k*(k-1).quo(2)) - df)*ln_n
+      end
+      def coefficients
+        est=Hash.new
+        coeffs=@r_summary['parameters']
+        # 0:name, 1:matrix, 2:row, 3:col, 4:estimate, 5:Std.error
+        coeffs[0].each_with_index do |v,i|
+          f1=coeffs[2][i]
+          f2=coeffs[3][i]
+          key=[f1,f2].sort
+          est[key]={:estimate=>coeffs[4][i], :se=>coeffs[5][i], :z=>nil, :p=>nil, :label=>v}
+        end
+        est
+        
+      end
+      dirty_memoize :chi_square, :df, :rmsea, :coefficients, :r_summary, :chi_square_null, :df_null, :nfi, :nnfi
+      
+    end
+  end
+end