pest 0.0.0

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+# uncomment this line if your project needs to run something other than `rake`:
+script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,13 @@
+source 'http://rubygems.org'
+
+gemspec
+
+gem "narray"
+gem "uuidtools"
+
+group :development do
+  gem "jeweler"
+  gem "rake"
+  gem "pry"
+  gem "rspec"
+end

data/README.md

@@ -0,0 +1,90 @@
+# Pest, a framework for Probability Estimation
+
+[![Build Status](https://secure.travis-ci.org/kerinin/pest.png)](http://travis-ci.org/kerinin/pest)
+
+
+Pest provides a unified framework for interacting with different probability
+estimation models.  
+
+* Pest tries to be agnostic about the underlying data data structures, 
+so changing libraries (GSL -> Hadoop) is as simple as using a different data source.
+* Pest is designed to create estimators using subsets of larger data sources, and
+transparently constructs estimators to facilitate dynamic querying
+* Implementing custom estimation models is easy, and Pest implements some model
+common ones for you.
+
+Pest abstracts common statstical operations including:
+
+* Marginal, Joint and Conditional point probability
+* Interval and Cumulative probability
+* Entropy, Cross Entropy, and Mutual Information
+* Mean, Median, Mode, etc
+
+
+## Ruby Install
+
+``` sh
+brew install gnuplot  # This may take awhile...
+cd /usr/local
+git checkout 83ed494 /usr/local/Library/Formula/gsl.rb
+brew install gsl      # Forcing gsl v1.4
+
+bundle install 
+```
+
+## API
+
+``` ruby
+# Creating Datasets
+test = Pest::DataSet::Hash.new hash                   # Creates a Hash dataset of observations from a hash
+test = Pest::DataSet::Hash.new file                   # Creates a Hash dataset of observations from an IO (Marshalled) 
+train = Pest::DataSet::GSL.new file                   # Creates a GSL dataset from and IO instance
+
+# DataSet Variables
+test.variables                                        # hash of Variable instances detected in observation set
+test.v                                                # alias of 'variables'
+test.v[:foo]                                          # a specific variable
+test.v[:foo] = another_variable                       # explicit declaration
+
+# Creating Estimators
+e = Pest::Estimator::Set::Multinomial.new(test)       # Creates a multinomial estimator for set o
+e = Pest::Estimator::Discrete::Gaussian.new(file)     # Creating an estimator with the DataSet API
+
+# Descriptive Statistical Properties
+e.mode(:foo)                                          # Mode
+e.mean(:foo)                                          # Mean (discrete & continuous only)
+e.median(:foo)                                        # Median (discrete & continuous only)
+# quantile?
+# variance?
+# deviation?
+
+# Estimating Entropy (Set & Discrete only)
+e.entropy(:foo)                                       # Entropy of 'foo'
+e.h(:foo, :bar)                                       # Joint entropy of 'foo' AND 'bar'
+e.h(:foo).given(:bar)                                 # Cross entropy of 'foo' : 'bar'
+e.mutual_information(:foo, :bar)                      # Mutual information of 'foo' and 'bar'
+e.i(:foo, :bar)                                       # Alias
+
+# Estimating Point Probability (Set & Discrete only)
+e.probability(o.variables[:foo])                      # (Set/Discrete only) Estimate the probability of all values of 'foo'
+e.p(:foo)                                             # Same as above, tries to find a variable named 'foo'
+e.p(:foo).in(test)                                    # Estimate the probability of values in dataset 'test'
+e.p(:foo).given(:bar).in(test)                        # Estimate the conditional foo | bar for the values in 'test'
+e.p(:foo, :bar).in(test)                              # Estimate the joint probablity foo AND bar
+e.p(:foo, :bar).given(:baz, :qux).in(test)            # More complex joint & conditional probabilities
+e.p(:foo => 4, :bar => 2).given(:baz => 0)            # Single prediction (implicitly creates dataset)
+e.p(:foo).given(:bar).cache                           # Builds and persists the model for 'foo|bar'
+e.p(:foo).given(:bar).cache('path.csv')               # Persist to a specific path (defaults to tmp)
+
+# Estimating Cumulative & Interval Probability (Discrete & Continuous only)
+e.probability(:foo).greater_than(:bar).in(test)
+e.p(:foo).greater_than(:bar).less_than(:baz).in(test)
+e.p(:foo).gt(:bar).lt(:baz).given(:qux).in(test)
+```
+
+## Working Notes
+
+Do we want variable equality to be name-based?  It may make more sense to allow
+variables named differently in different data sets to be equivalent. And how the
+fuck do we handle variable type?  I'm almost thinking we don't, and let the actual
+estimators take care of type casting

data/Rakefile

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "pest"
+  gem.homepage = "http://github.com/kerinin/pest"
+  gem.license = "MIT"
+  gem.description = %q{Wrappers to facilitate different classes of probability estimators}
+  gem.summary = %q{Probability Estimation}
+  gem.email = "kerinin@gmail.com"
+  gem.authors = ["Ryan Michael"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+# require 'rcov/rcovtask'
+# Rcov::RcovTask.new do |test|
+#   test.libs << 'test'
+#   test.pattern = 'test/**/test_*.rb'
+#   test.verbose = true
+#   test.rcov_opts << '--exclude "gems/*"'
+# end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "pest #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/lib/pest.rb

@@ -0,0 +1,23 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
+
+require 'uuidtools'
+require 'csv'
+
+require "pest/version"
+require "pest/variable"
+
+require "pest/function"
+require "pest/function/probability"
+require "pest/function/entropy"
+
+require "pest/data_set"
+require "pest/data_set/hash"
+require "pest/data_set/narray"
+
+require "pest/estimator"
+require "pest/estimator/frequency"
+
+module Pest
+  CACHE_TO_FILE = false
+end

data/lib/pest/data_set.rb

@@ -0,0 +1,62 @@
+module Pest::DataSet
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  def variables
+    @variables ||= {}
+  end
+
+  def variable_array
+    @variables.values.sort
+  end
+
+  def to_hash(*args)
+    raise NotImplementedError
+  end
+
+  def save(*args)
+    raise NotImplementedError
+  end
+
+  def destroy
+    raise NotImplementedError
+  end
+
+  def length
+    raise NotImplementedError
+  end
+
+  module ClassMethods
+    def from(data_source)
+      # Try to translate the data source directly
+      if translator_method = translators[data_source.class]
+        send(translator_method, data_source)
+
+      # Try to translate via hash
+      else
+        begin
+          hash_data = data_source.to_hash
+        rescue NoMethodError
+          raise "Unrecognized data source type"
+        end
+        
+        if hash_data and translators.has_key?(hash_data.class)
+          from(data_source.to_hash)
+        end
+      end
+    end
+
+    def translators(*args)
+      raise NotImplementedError
+    end
+
+    def from_file(*args)
+      raise NotImplementedError
+    end
+
+    def from_hash(*args)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/pest/data_set/hash.rb

@@ -0,0 +1,70 @@
+class Pest::DataSet::Hash
+  include Pest::DataSet
+
+  def self.translators
+    {
+      File    => :from_file,
+      String  => :from_file,
+      Symbol  => :from_file
+    }
+  end
+
+  def self.from_file(file)
+    file = File.open(file.to_s, 'r') if file.kind_of?(String)
+
+    object = Marshal.restore(file)
+
+    if object.kind_of?(::Hash)
+      self.new(object)
+    else
+      raise "File does not seem to contain valid data"
+    end
+  end
+
+  attr_reader :variables, :hash
+
+  def initialize(hash)
+    @hash = hash
+    @variables = {}
+    hash.keys().each do |name|
+      @variables[name] = Pest::Variable.new(:name => name)
+    end
+  end
+
+  def to_hash
+    @hash
+  end
+
+  def data_vectors(variables=nil)
+    VectorEnumerable.new(self,variables)
+  end
+
+  def length
+    @hash.values.first.length
+  end
+
+  def save(file=nil)
+    file ||= Tempfile.new('pest_hash_dataset')
+    file = File.open(file, 'w') if file.kind_of?(String)
+    Marshal.dump(@hash, file)
+  end
+
+  class VectorEnumerable
+    include Enumerable
+
+    def initialize(data_set,variables=nil)
+      @data_set = data_set
+      @variables = variables || @data_set.variables
+    end
+
+    def [](i)
+      @variables.map {|var| @data_set.hash[var][i]}
+    end
+
+    def each
+      @data_set.hash.values.first.each_index do |i|
+        yield @variables.keys.map {|var| @data_set.hash[var][i]}
+      end
+    end
+  end
+end

data/lib/pest/data_set/narray.rb

@@ -0,0 +1,98 @@
+require 'narray'
+
+class Pest::DataSet::NArray < NMatrix
+  include Pest::DataSet
+
+  def self.translators
+    {
+      Hash    => :from_hash,
+      File    => :from_file,
+      String  => :from_file,
+      Symbol  => :from_file
+    }
+  end
+
+  def self.from_hash(hash)
+    data_set = to_na(hash.keys.sort.map {|key| hash[key]}) # Ensure the matrix is sorted the same as the variables
+    data_set.variables = {}
+    hash.keys.each do |key|
+      variable = key.kind_of?(Pest::Variable) ? key : Pest::Variable.new(:name => key)
+      data_set.variables[variable.name] = variable
+    end
+    data_set
+  end
+
+  def self.from_file(file)
+    file = File.open(file.to_s, 'r') if file.kind_of?(String)
+
+    begin
+      variables, matrix = Marshal.restore(file)
+      data_set = to_na(matrix)
+      data_set.variables = variables
+      data_set
+    rescue
+      raise "File does not seem to contain valid data"
+    end
+  end
+
+  def self.from_csv(file, args={})
+    args = args.merge({:converters => :all})
+    data = CSV.read(file, args)
+    data_set = to_na(data[1..-1]).transpose
+    data_set.variables = {}
+    data[0].each do |key|
+      variable = key.kind_of?(Pest::Variable) ? key : Pest::Variable.new(:name => key)
+      data_set.variables[variable.name] = variable
+    end
+    data_set
+  end
+
+  attr_accessor :variables
+
+  def to_hash
+    hash = {}
+    variables.values.each_index do |i|
+      hash[variables.values[i]] = self[true,i].to_a[0]
+    end
+    hash
+  end
+
+  # variables: an array of variables for which each vector should contain values
+  # Order is retained in the returned value
+  def data_vectors(variables=nil)
+    VectorEnumerable.new(self, variables)
+  end
+
+  def length
+    shape[0]
+  end
+
+  def save(file=nil)
+    file ||= Tempfile.new('pest_hash_dataset')
+    file = File.open(file, 'w') if file.kind_of?(String)
+    Marshal.dump([variables,to_a], file)
+    file.close
+  end
+
+  class VectorEnumerable
+    include Enumerable
+
+    def initialize(data_set, variables = true)
+      @data_set = data_set
+      @variables = variables
+      if @variables.kind_of?(Enumerable)
+        @variables = variables.map {|v| @data_set.variable_array.index(v)}
+      end
+    end
+
+    def [](i)
+      @data_set[i,@variables].transpose
+    end
+
+    def each
+      (0..@data_set.shape[0]-1).each do |i|
+        yield Array(self[i]).first
+      end
+    end
+  end
+end

data/lib/pest/estimator.rb

@@ -0,0 +1,72 @@
+module Pest::Estimator 
+  attr_accessor :data
+
+  def initialize(data=nil)
+    @data = data
+  end
+
+  def variables
+    @data.nil? ? {} : @data.variables
+  end
+
+  def distributions
+    @distributions ||= DistributionList.new(self)
+  end
+
+  def to_variable(arg)
+    variable = case arg.class.name
+    when 'Pest::Variable'
+      arg
+    when 'String', 'Symbol'
+      variables[arg] || Pest::Variable.new(:name => arg)
+    end
+    raise ArgumentError unless variables.values.include?(variable)
+    variable
+  end
+
+  module Distribution
+    attr_reader :variables
+
+    def initialize(estimator, variables)
+      @estimator = estimator
+      @variables = variables
+    end
+
+    def variable_array
+      variables.to_a.sort
+    end
+
+    def probability
+      raise NotImplementedError
+    end
+  end
+
+  class DistributionList < Hash
+    def initialize(estimator)
+      @estimator = estimator
+    end
+
+    def parse_args(args)
+      set = if args.kind_of? Array
+        if args.any? {|arg| arg.kind_of?(::Set)}
+          args.inject(::Set.new) {|set, el| set + el.to_set}
+        else
+          args.flatten.to_set
+        end
+      elsif args.kind_of? ::Set
+        args
+      else
+        Array(args).to_set
+      end
+      set.map! {|arg| @estimator.to_variable(arg) }
+    end
+
+    def [](*args)
+      set = parse_args(args)
+      unless has_key? set
+        self[set] = @estimator.distribution_class.new(@estimator, set)
+      end
+      super(set)
+    end
+  end
+end