cross_validation 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e36ebf38a97ebf8665474186f2c1ae3c82a3855d
+  data.tar.gz: 770c131f2d8e3cf359e7e26450cb7a24c704239c
+SHA512:
+  metadata.gz: 34ce7b4484db03a2d09aeb2b1f40c586a84a63b6ba22d355220ee6cba8bf588f3c8a2ac867a82524d43aa9fba06863f4f44ca09af8a17cb7f56409d9105dce7f
+  data.tar.gz: d55546305d845c1f2c977f825f2d69a77b32369c3afc13b5262dd170b523f4efffa6f7fa9f6d9f7d9215b6b008156ff1ae2cb13993625268bc8d3e1f0b4b2f4c

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.bin

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cross_validation.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Jon-Michael Deldin
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# CrossValidation
+
+[![Build Status](https://travis-ci.org/jmdeldin/cross_validation.png?branch=master)](https://travis-ci.org/jmdeldin/cross_validation)
+[![Code Climate](https://codeclimate.com/github/jmdeldin/cross_validation.png)](https://codeclimate.com/github/jmdeldin/cross_validation)
+
+This gem provides a k-fold cross-validation routine and confusion matrix
+for evaluating machine learning classifiers.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'cross_validation'
+
+And then execute:
+
+    $ bundle install --binstubs .bin
+
+Or install it yourself as:
+
+    $ gem install cross_validation
+
+## Usage
+
+Cross-validation:
+
+Confusion-matrix:
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Questions
+
+Send me an email, `dev@jmdeldin.com`

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/test_*']
+  t.verbose = true
+end
+
+task :default => :test

data/cross_validation.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'cross_validation'
+
+Gem::Specification.new do |gem|
+  gem.name          = "cross_validation"
+  gem.version       = CrossValidation::VERSION
+  gem.authors       = ["Jon-Michael Deldin"]
+  gem.email         = ["dev@jmdeldin.com"]
+  gem.summary       = %q{Performs k-fold cross-validation on machine learning
+                      classifiers.}
+  gem.description   = gem.summary
+  gem.homepage      = 'https://github.com/jmdeldin/cross_validation'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency('rake', '~> 10.0')
+end

data/lib/cross_validation.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+module CrossValidation
+  VERSION = '0.0.1'
+end

data/lib/cross_validation/confusion_matrix.rb

@@ -0,0 +1,100 @@
+require_relative '../cross_validation'
+
+module CrossValidation
+
+  # Provides a confusion matrix (contingency table) for classification
+  # results.
+  #
+  # See the following book for more details:
+  #
+  # Speech and Language Processing: An introduction to natural language
+  # processing, computational linguistics, and speech recognition. Daniel
+  # Jurafsky & James H. Martin.
+  class ConfusionMatrix
+    # Initialize the confusion matrix with a Proc (or block). This Proc must
+    # return a symbol of :tp (true positive), :tn (true negative), :fp (false
+    # positive), or :fn (false negative) for a given classification and its
+    # expected value.
+    #
+    # See the unit test for an example Proc.
+    #
+    # @param [Proc] keys_proc
+    def initialize(keys_proc)
+      @keys_for = keys_proc
+      @values = {:tp => 0, :tn => 0, :fp => 0, :fn => 0}
+    end
+
+    [:tp, :tn, :fp, :fn].each do |field|
+      define_method(field) { @values.fetch(field) }
+    end
+
+    # Save the result of classification
+    #
+    # @param [Object] actual  The classified value
+    # @param [Object] truth   The known, expected value
+    # @return [self]
+    def store(actual, truth)
+      key = @keys_for.call(actual, truth)
+
+      if @values.key?(key)
+        @values[key] += 1
+      else
+        fail IndexError, "#{key} not found in confusion matrix"
+      end
+
+      self
+    end
+
+    # Computes the accuracy of the classifier, defined as (tp + tn)/n
+    #
+    # @return [Float]
+    def accuracy
+      (@values.fetch(:tp) + @values.fetch(:tn)) / total()
+    end
+
+    # Computes the precision of the classifier, defined as tp/(tp + fp)
+    #
+    # @return [Float]
+    def precision
+      @values.fetch(:tp) / Float(@values.fetch(:tp) + @values.fetch(:fp))
+    end
+
+    # Computes the recall of the classifier, defined as tp/(tp + fn)
+    #
+    # @return [Float]
+    def recall
+      @values.fetch(:tp) / Float(@values.fetch(:tp) + @values.fetch(:fn))
+    end
+
+    # Returns the F-measure of the classifier's precision and recall.
+    #
+    # @param [Float] beta Favor precision (<1), recall (>1), or both (1)
+    # @return [Float]
+    def fscore(beta)
+      b2 = Float(beta**2)
+      ((b2 + 1) * precision * recall) / (b2 * precision + recall)
+    end
+
+    # Returns an F-score that favors precision and recall equally.
+    #
+    # @return [Float]
+    def f1
+      fscore(1)
+    end
+
+    # Returns the classifier's error
+    def error
+      1.0 - accuracy()
+    end
+
+    private
+
+    # Returns the total number of classifications as a Float, since this value
+    # is used as a divisor.
+    #
+    # @return [Float]
+    def total
+      Float(@values.values.reduce(:+))
+    end
+  end
+end

data/lib/cross_validation/runner.rb

@@ -0,0 +1,95 @@
+require_relative '../cross_validation'
+
+module CrossValidation
+  class Runner
+    # @return [Array] Array of documents to train and test on. It can be an
+    #                 array of anything, as the +fetch_sample_value+ and
+    #                 +fetch_sample_class+ lambdas specify what to feed into
+    #                 the classifying method.
+    attr_accessor :documents
+
+    # @return [Proc] This instantiates your classifier.
+    attr_accessor :classifier
+
+    # @return [Fixnum] The number of folds to partition +documents+ into.
+    #                  Mutually exclusive with +percentage+.
+    attr_accessor :folds
+
+    # @return [Float] The number of folds to partition +documents+ into as a
+    #                 *percentage* of the documents. Mutually exclusive with
+    #                 +folds+.
+    attr_accessor :percentage
+
+    # @return [ConfusionMatrix]
+    attr_accessor :matrix
+
+    # @return [Proc] This receives an instantiated +classifier+ and a
+    #                document, and it should call your classifier's training
+    #                method.
+    attr_accessor :training
+
+    # @return [Proc] This receives a *trained* classifier and a test document.
+    #                It classifies the document. It's a +Proc+ because we
+    #                create a new one with each partition.
+    attr_accessor :classifying
+
+    # @return [Proc] This receives a document and should return its value,
+    #                i.e., whatever you're feeding into +classifying+.
+    attr_accessor :fetch_sample_value
+
+    # @return [Proc] When verifying the results of executing the +classifying+
+    #                method, we need to determine what the actual class (e.g.,
+    #                spam) of the document was. This +Proc+ receives a
+    #                document and should return the document's class.
+    attr_accessor :fetch_sample_class
+
+    # Returns the number of folds to partition the documents into.
+    #
+    # @return [Fixnum]
+    def k
+      @k ||= percentage ? (documents.size * percentage) : folds
+    end
+
+    # Performs k-fold cross-validation and returns a confusion matrix.
+    #
+    # The algorithm is as follows (Mitchell, 1997, p147):
+    #
+    #   partitions = partition data into k-equal sized subsets (folds)
+    #   for i = 1 -> k:
+    #     T = data \ partitions[i]
+    #     train(T)
+    #     classify(partitions[i])
+    #   output confusion matrix
+    #
+    def run
+      partitions = documents.each_slice(k).to_a
+
+      results = partitions.map.with_index do |part, i|
+        # Array#rotate puts the element i first, so all we have to do is rotate
+        # then remove that element to get the training set. Array#drop does not
+        # mutate the original array either. Array#flatten is needed to coalesce
+        # our list of lists into one list again.
+        training_samples = partitions.rotate(i).drop(1).flatten
+
+        classifier_instance = classifier.call()
+
+        # train it
+        training_samples.each { |doc| training.call(classifier_instance, doc) }
+
+        # fetch confusion keys
+        part.each do |x|
+          prediction = classifying.call(classifier_instance, fetch_sample_value.call(x))
+          matrix.store(prediction, fetch_sample_class.call(x))
+        end
+      end
+
+      matrix
+    end
+
+    # Configuring a cross-validation run is complicated. Let's make it easier
+    # with a factory method.
+    def self.create
+      new.tap { |r| yield(r) }
+    end
+  end
+end

data/test/test_confusion_matrix.rb

@@ -0,0 +1,101 @@
+require_relative 'test_helper'
+require_relative '../lib/cross_validation/confusion_matrix'
+
+class TestConfusionMatrix < MiniTest::Unit::TestCase
+  def delta
+    1e-6
+  end
+
+  def setup
+    @mat = CrossValidation::ConfusionMatrix.new(method(:keys_for))
+  end
+
+  def test_true_positives
+    true_positive(@mat)
+    assert_equal 1, @mat.tp
+  end
+
+  def test_true_negatives
+    true_negative(@mat)
+    assert_equal 1, @mat.tn
+  end
+
+  def test_false_positives
+    false_positive(@mat)
+    assert_equal 1, @mat.fp
+  end
+
+  def test_false_negatives
+    false_negative(@mat)
+    assert_equal 1, @mat.fn
+  end
+
+  def test_store_raises_index_error_on_bad_key
+    bad_keys_for = ->(actual, expected) { :bad }
+    mat = CrossValidation::ConfusionMatrix.new(bad_keys_for)
+    assert_raises IndexError do
+      mat.store(:ham, :spam)
+    end
+  end
+
+  def test_accuracy
+    true_positive(@mat)
+    true_negative(@mat)
+    false_negative(@mat)
+
+    assert_in_delta 2.0/3.0, @mat.accuracy, delta
+  end
+
+  def test_precision
+    true_positive(@mat)
+    false_positive(@mat)
+
+    assert_in_delta 0.5, @mat.precision, delta
+  end
+
+  def test_error
+    true_positive(@mat)
+    true_negative(@mat)
+    false_positive(@mat)
+
+    assert_in_delta 1/3.0, @mat.error, delta
+  end
+
+  def test_precision
+    true_positive(@mat)
+    false_positive(@mat)
+    false_positive(@mat)
+    false_positive(@mat)
+
+    assert_in_delta 0.25, @mat.precision, delta
+  end
+
+  def test_recall
+    true_positive(@mat)
+    false_negative(@mat)
+
+    assert_in_delta 0.5, @mat.recall, delta
+  end
+
+  def test_fscore
+    true_positive(@mat)
+    true_negative(@mat)
+    false_positive(@mat)
+
+    assert_in_delta 2/3.0, @mat.fscore(1), delta
+  end
+
+  def test_f1score
+    true_positive(@mat)
+    true_negative(@mat)
+
+    assert_in_delta 1.0, @mat.f1, delta
+  end
+
+  private
+
+  def true_positive(mat)  mat.store(:spam, :spam) end
+  def true_negative(mat)  mat.store(:ham, :ham)   end
+  def false_positive(mat) mat.store(:ham, :spam)  end
+  def false_negative(mat) mat.store(:spam, :ham)  end
+end

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+require 'minitest/autorun'
+
+# Dummy method for use in testing confusion matrices.
+def keys_for(actual, expected)
+  if actual == :spam
+    expected == :spam ? :tp : :fn
+  elsif actual == :ham
+    expected == :ham ? :tn : :fp
+  end
+end

data/test/test_runner.rb

@@ -0,0 +1,86 @@
+require_relative 'test_helper'
+require_relative '../lib/cross_validation/confusion_matrix'
+require_relative '../lib/cross_validation/runner'
+
+# A stupid classifier
+class SpamClassifier
+  def train(klass, document)
+    # don't bother, we're that good (in reality, you should probably do some
+    # work here)
+  end
+
+  def classify(document)
+    document =~ /viagra/ ? :spam : :ham
+  end
+end
+
+# We just need to associate a class with a value. Feel free to use whatever
+# data structure you like -- this is only used in user-defined training and
+# classifying closures.
+Sample = Struct.new(:klass, :value)
+
+# Asserts the DSL's getter and setters work.
+def check_dsl(attribute, value)
+  runner = CrossValidation::Runner.create { |r|
+    r.public_send("#{attribute}=", :value)
+  }
+
+  define_method("test_#{attribute}_getter") {
+    assert_equal :value, runner.public_send(attribute)
+  }
+end
+
+class TestRunner < MiniTest::Unit::TestCase
+  def setup
+    tpl = ['Buy some...', 'Would you like some...']
+    @spam = tpl.map { |pfx| Sample.new(:spam, pfx + 'viagra!') }
+    @ham = tpl.map { |pfx| Sample.new(:ham, pfx + 'penicillin!') }
+    @corpus = @spam + @ham
+    @corpus *= 25 # 100 is easier to deal with
+  end
+
+  def test_run
+    runner = CrossValidation::Runner.create do |r|
+      r.documents = @corpus
+      r.folds = 10
+      r.classifier = lambda { SpamClassifier.new }
+      r.fetch_sample_class = lambda { |sample| sample.klass }
+      r.fetch_sample_value = lambda { |sample| sample.value }
+      r.matrix = CrossValidation::ConfusionMatrix.new(method(:keys_for))
+      r.training = lambda { |classifier, doc|
+        classifier.train doc.klass, doc.value
+      }
+      r.classifying = lambda { |classifier, doc|
+        classifier.classify doc
+      }
+    end
+
+    mat = runner.run
+
+    assert_equal 50, mat.tp
+    assert_equal 50, mat.tn
+  end
+
+  def test_percentage_takes_precedence_over_folds
+    runner = CrossValidation::Runner.create do |r|
+      r.documents = ['foo'] * 100
+      r.folds = 20
+      r.percentage = 0.1
+    end
+
+    assert_equal 10, runner.k
+  end
+
+  [
+   :documents,
+   :folds,
+   :classifier,
+   :fetch_sample_value,
+   :fetch_sample_class,
+   :matrix,
+   :training,
+   :classifying,
+  ].each do |attribute|
+    check_dsl(attribute, :foo)
+  end
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: cross_validation
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jon-Michael Deldin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Performs k-fold cross-validation on machine learning classifiers.
+email:
+- dev@jmdeldin.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- cross_validation.gemspec
+- lib/cross_validation.rb
+- lib/cross_validation/confusion_matrix.rb
+- lib/cross_validation/runner.rb
+- test/test_confusion_matrix.rb
+- test/test_helper.rb
+- test/test_runner.rb
+homepage: https://github.com/jmdeldin/cross_validation
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0.rc.2
+signing_key: 
+specification_version: 4
+summary: Performs k-fold cross-validation on machine learning classifiers.
+test_files:
+- test/test_confusion_matrix.rb
+- test/test_helper.rb
+- test/test_runner.rb