classifier-fork 1.3.4

data/README.markdown

@@ -0,0 +1,97 @@
+## Welcome to Classifier
+
+Classifier is a general module to allow Bayesian and other types of classifications.
+
+## Download
+
+* https://github.com/cardmagic/classifier
+* gem install classifier
+* git clone https://github.com/cardmagic/classifier.git
+
+## Dependencies
+
+If you install Classifier from source, you'll need to install Roman Shterenzon's fast-stemmer gem with RubyGems as follows:
+
+    gem install fast-stemmer
+
+If you would like to speed up LSI classification by at least 10x, please install the following libraries:
+GNU GSL:: http://www.gnu.org/software/gsl
+rb-gsl:: http://rb-gsl.rubyforge.org
+
+Notice that LSI will work without these libraries, but as soon as they are installed, Classifier will make use of them. No configuration changes are needed, we like to keep things ridiculously easy for you.
+
+## Bayes
+
+A Bayesian classifier by Lucas Carlson. Bayesian Classifiers are accurate, fast, and have modest memory requirements.
+
+### Usage
+
+    require 'classifier'
+    b = Classifier::Bayes.new 'Interesting', 'Uninteresting'
+    b.train_interesting "here are some good words. I hope you love them"
+    b.train_uninteresting "here are some bad words, I hate you"
+    b.classify "I hate bad words and you" # returns 'Uninteresting'
+    
+    require 'madeleine'
+    m = SnapshotMadeleine.new("bayes_data") {
+        Classifier::Bayes.new 'Interesting', 'Uninteresting'
+    }
+    m.system.train_interesting "here are some good words. I hope you love them"
+    m.system.train_uninteresting "here are some bad words, I hate you"
+    m.take_snapshot
+    m.system.classify "I love you" # returns 'Interesting'
+
+Using Madeleine, your application can persist the learned data over time.
+
+### Bayesian Classification
+
+* http://www.process.com/precisemail/bayesian_filtering.htm
+* http://en.wikipedia.org/wiki/Bayesian_filtering
+* http://www.paulgraham.com/spam.html
+
+## LSI
+
+A Latent Semantic Indexer by David Fayram. Latent Semantic Indexing engines
+are not as fast or as small as Bayesian classifiers, but are more flexible, providing 
+fast search and clustering detection as well as semantic analysis of the text that 
+theoretically simulates human learning.
+
+### Usage
+
+    require 'classifier'
+    lsi = Classifier::LSI.new
+    strings = [ ["This text deals with dogs. Dogs.", :dog],
+              ["This text involves dogs too. Dogs! ", :dog],
+              ["This text revolves around cats. Cats.", :cat],
+              ["This text also involves cats. Cats!", :cat],
+              ["This text involves birds. Birds.",:bird ]]
+    strings.each {|x| lsi.add_item x.first, x.last}
+  
+    lsi.search("dog", 3)
+    # returns => ["This text deals with dogs. Dogs.", "This text involves dogs too. Dogs! ", 
+    #             "This text also involves cats. Cats!"]
+
+    lsi.find_related(strings[2], 2)
+    # returns => ["This text revolves around cats. Cats.", "This text also involves cats. Cats!"]
+  
+    lsi.classify "This text is also about dogs!"
+    # returns => :dog
+  
+Please see the Classifier::LSI documentation for more information. It is possible to index, search and classify
+with more than just simple strings. 
+
+### Latent Semantic Indexing
+
+* http://www.c2.com/cgi/wiki?LatentSemanticIndexing
+* http://www.chadfowler.com/index.cgi/Computing/LatentSemanticIndexing.rdoc
+* http://en.wikipedia.org/wiki/Latent_semantic_analysis
+
+## Authors    
+
+* Lucas Carlson  (lucas@rufy.com)
+* David Fayram II (dfayram@gmail.com)
+* Cameron McBride (cameron.mcbride@gmail.com)
+* Ivan Acosta-Rubio (ivan@softwarecriollo.com)
+
+This library is released under the terms of the GNU LGPL. See LICENSE for more details.
+

data/Rakefile

@@ -0,0 +1,97 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rdoc/task'
+require 'rubygems/package_task'
+require 'rake/contrib/rubyforgepublisher'
+
+PKG_VERSION = "1.3.4"
+
+PKG_FILES = FileList[
+    "lib/**/*", "bin/*", "test/**/*", "[A-Z]*", "Rakefile", "Gemfile", "html/**/*"
+]
+
+desc "Default Task"
+task :default => [ :test ]
+
+# Run the unit tests
+desc "Run all unit tests"
+Rake::TestTask.new("test") { |t|
+  t.libs << "lib"
+  t.pattern = 'test/*/*_test.rb'
+  t.verbose = true
+}
+
+# Make a console, useful when working on tests
+desc "Generate a test console"
+task :console do
+   verbose( false ) { sh "irb -I lib/ -r 'classifier'" }
+end
+
+# Genereate the RDoc documentation
+desc "Create documentation"
+Rake::RDocTask.new("doc") { |rdoc|
+  rdoc.title = "Ruby Classifier - Bayesian and LSI classification library"
+  rdoc.rdoc_dir = 'html'
+  rdoc.rdoc_files.include('README.markdown')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+
+# Genereate the package
+spec = Gem::Specification.new do |s|
+
+  #### Basic information.
+
+  s.name = 'classifier'
+  s.version = PKG_VERSION
+  s.version = "#{s.version}-alpha-#{ENV['TRAVIS_BUILD_NUMBER']}" if ENV['TRAVIS']
+  s.summary = <<-EOF
+   A general classifier module to allow Bayesian and other types of classifications.
+  EOF
+  s.description = <<-EOF
+   A general classifier module to allow Bayesian and other types of classifications.
+  EOF
+
+  #### Which files are to be included in this gem?  Everything!  (Except CVS directories.)
+
+  s.files = PKG_FILES
+
+  #### Load-time details: library and application (you will need one or both).
+
+  s.require_path = 'lib'
+  s.autorequire = 'classifier'
+
+  #### Documentation and testing.
+
+  s.has_rdoc = true
+
+  #### Dependencies and requirements.
+
+  s.add_dependency('fast-stemmer', '>= 1.0.0')
+  s.requirements << "A porter-stemmer module to split word stems."
+
+  #### Author and project details.
+  s.author = "Lucas Carlson"
+  s.email = "lucas@rufy.com"
+  s.homepage = "http://classifier.rufy.com/"
+end
+
+Gem::PackageTask.new(spec) do |pkg|
+  pkg.need_zip = true
+  pkg.need_tar = true
+end
+
+desc "Report code statistics (KLOCs, etc) from the application"
+task :stats do
+  require 'code_statistics'
+  CodeStatistics.new(
+    ["Library", "lib"],
+    ["Units", "test"]
+  ).to_s
+end
+
+desc "Publish new documentation"
+task :publish do
+   `ssh rufy update-classifier-doc`
+    Rake::RubyForgePublisher.new('classifier', 'cardmagic').upload
+end

data/bin/bayes.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+begin
+	require 'rubygems'
+	require 'classifier'
+rescue
+	require 'classifier'
+end
+
+require 'madeleine'
+
+m = SnapshotMadeleine.new(File.expand_path("~/.bayes_data")) {
+	Classifier::Bayes.new 'Interesting', 'Uninteresting'
+}
+
+case ARGV[0]
+when "add"
+	case ARGV[1].downcase
+	when "interesting"
+		m.system.train_interesting File.open(ARGV[2]).read
+		puts "#{ARGV[2]} has been classified as interesting"
+	when "uninteresting"
+		m.system.train_uninteresting File.open(ARGV[2]).read
+		puts "#{ARGV[2]} has been classified as uninteresting"
+	else
+		puts "Invalid category: choose between interesting and uninteresting"
+		exit(1)
+	end
+when "classify"
+	puts m.system.classify(File.open(ARGV[1]).read)
+else
+	puts "Invalid option: choose add [category] [file] or clasify [file]"
+	exit(-1)
+end
+
+m.take_snapshot

data/bin/summarize.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+begin
+        require 'rubygems'
+        require 'classifier'
+rescue
+        require 'classifier'
+end
+
+require 'open-uri'
+
+num = ARGV[1].to_i
+num = num < 1 ? 10 : num
+
+text = open(ARGV.first).read
+puts text.gsub(/<[^>]+>/,"").gsub(/[\s]+/," ").summary(num)

data/lib/classifier.rb

@@ -0,0 +1,30 @@
+#--
+# Copyright (c) 2005 Lucas Carlson
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require 'rubygems'
+require 'classifier/extensions/string'
+require 'classifier/bayes'
+require 'classifier/lsi'

data/lib/classifier/bayes.rb

@@ -0,0 +1,135 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+module Classifier
+
+class Bayes
+  # The class can be created with one or more categories, each of which will be
+  # initialized and given a training method. E.g., 
+  #      b = Classifier::Bayes.new 'Interesting', 'Uninteresting', 'Spam'
+	def initialize(*categories)
+		@categories = Hash.new
+		categories.each { |category| @categories[category.prepare_category_name] = Hash.new }
+		@total_words = 0
+                @category_counts = Hash.new(0)
+	end
+
+	#
+	# Provides a general training method for all categories specified in Bayes#new
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train :this, "This text"
+	#     b.train "that", "That text"
+	#     b.train "The other", "The other text"
+	def train(category, text)
+		category = category.prepare_category_name
+                @category_counts[category] += 1
+		text.word_hash.each do |word, count|
+			@categories[category][word]     ||=     0
+			@categories[category][word]      +=     count
+			@total_words += count
+		end
+	end
+
+	#
+	# Provides a untraining method for all categories specified in Bayes#new
+	# Be very careful with this method.
+	#
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train :this, "This text"
+	#     b.untrain :this, "This text"
+	def untrain(category, text)
+		category = category.prepare_category_name
+                @category_counts[category] -= 1
+		text.word_hash.each do |word, count|
+			if @total_words >= 0
+				orig = @categories[category][word]
+				@categories[category][word]     ||=     0
+				@categories[category][word]      -=     count
+				if @categories[category][word] <= 0
+					@categories[category].delete(word)
+					count = orig
+				end
+				@total_words -= count
+			end
+		end
+	end
+		
+	#
+	# Returns the scores in each category the provided +text+. E.g.,
+	#    b.classifications "I hate bad words and you"
+	#    =>  {"Uninteresting"=>-12.6997928013932, "Interesting"=>-18.4206807439524}
+	# The largest of these scores (the one closest to 0) is the one picked out by #classify
+	def classifications(text)
+		score = Hash.new
+                training_count = @category_counts.values.inject { |x,y| x+y }.to_f
+		@categories.each do |category, category_words|
+			score[category.to_s] = 0
+			total = category_words.values.inject(0) {|sum, element| sum+element}
+			text.word_hash.each do |word, count|
+				s = category_words.has_key?(word) ? category_words[word] : 0.1
+				score[category.to_s] += Math.log(s/total.to_f)
+			end
+                        # now add prior probability for the category
+                        s = @category_counts.has_key?(category) ? @category_counts[category] : 0.1
+                        score[category.to_s] += Math.log(s / training_count)
+		end
+		return score
+	end
+
+  #
+  # Returns the classification of the provided +text+, which is one of the 
+  # categories given in the initializer. E.g.,
+  #    b.classify "I hate bad words and you"
+  #    =>  'Uninteresting'
+	def classify(text)
+		(classifications(text).sort_by { |a| -a[1] })[0][0]
+	end
+	
+	#
+	# Provides training and untraining methods for the categories specified in Bayes#new
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train_this "This text"
+	#     b.train_that "That text"
+	#     b.untrain_that "That text"
+	#     b.train_the_other "The other text"
+	def method_missing(name, *args)
+		category = name.to_s.gsub(/(un)?train_([\w]+)/, '\2').prepare_category_name
+		if @categories.has_key? category
+			args.each { |text| eval("#{$1}train(category, text)") }
+		elsif name.to_s =~ /(un)?train_([\w]+)/
+			raise StandardError, "No such category: #{category}"
+		else
+	    super  #raise StandardError, "No such method: #{name}"
+		end
+	end
+	
+	#
+	# Provides a list of category names
+	# For example:
+	#     b.categories
+	#     =>   ['This', 'That', 'the_other']
+	def categories # :nodoc:
+		@categories.keys.collect {|c| c.to_s}
+	end
+	
+	#
+	# Allows you to add categories to the classifier.
+	# For example:
+	#     b.add_category "Not spam"
+	#
+	# WARNING: Adding categories to a trained classifier will
+	# result in an undertrained category that will tend to match
+	# more criteria than the trained selective categories. In short,
+	# try to initialize your categories at initialization.
+	def add_category(category)
+		@categories[category.prepare_category_name] = Hash.new
+	end
+	
+	alias append_category add_category
+end
+
+end

data/lib/classifier/extensions/string.rb

@@ -0,0 +1,10 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require 'fast_stemmer'
+require 'classifier/extensions/word_hash'
+
+class Object
+	def prepare_category_name; to_s.gsub("_"," ").capitalize.intern end
+end

data/lib/classifier/extensions/vector.rb

@@ -0,0 +1,112 @@
+# Author::    Ernest Ellingson
+# Copyright:: Copyright (c) 2005 
+
+# These are extensions to the std-lib 'matrix' to allow an all ruby SVD
+
+require 'matrix'
+require 'mathn'
+
+class Array
+  def sum(identity = 0, &block)
+    return identity unless size > 0
+  
+    if block_given?
+      map(&block).sum
+    else
+      reduce(:+)
+    end
+  end
+end
+
+class Vector
+  def magnitude
+    sumsqs = 0.0
+    self.size.times do |i|
+      sumsqs += self[i] ** 2.0 
+    end
+    Math.sqrt(sumsqs)
+  end
+  def normalize
+    nv = []
+    mag = self.magnitude
+    self.size.times do |i|
+
+      nv << (self[i] / mag)
+
+    end
+    Vector[*nv]
+  end
+end
+
+class Matrix
+  def Matrix.diag(s)
+     Matrix.diagonal(*s)
+  end
+  
+  alias :trans :transpose
+
+  def SV_decomp(maxSweeps = 20)
+    if self.row_size >= self.column_size
+      q = self.trans * self
+    else
+      q = self * self.trans
+    end
+    
+    qrot    = q.dup
+    v       = Matrix.identity(q.row_size)
+    azrot   = nil
+    mzrot   = nil
+    cnt     = 0
+    s_old   = nil
+    mu      = nil
+
+    while true do
+      cnt += 1
+      for row in (0...qrot.row_size-1) do
+        for col in (1..qrot.row_size-1) do
+          next if row == col
+          h = Math.atan((2 * qrot[row,col])/(qrot[row,row]-qrot[col,col]))/2.0
+          hcos = Math.cos(h)
+          hsin = Math.sin(h)
+          mzrot = Matrix.identity(qrot.row_size)
+          mzrot[row,row] = hcos
+          mzrot[row,col] = -hsin
+          mzrot[col,row] = hsin
+          mzrot[col,col] = hcos
+          qrot = mzrot.trans * qrot * mzrot
+          v = v * mzrot
+        end 
+      end
+      s_old = qrot.dup if cnt == 1
+      sum_qrot = 0.0 
+      if cnt > 1
+        qrot.row_size.times do |r|
+          sum_qrot += (qrot[r,r]-s_old[r,r]).abs if (qrot[r,r]-s_old[r,r]).abs > 0.001
+        end
+        s_old = qrot.dup
+      end 
+      break if (sum_qrot <= 0.001 and cnt > 1) or cnt >= maxSweeps
+    end # of do while true
+    s = []
+    qrot.row_size.times do |r|
+      s << Math.sqrt(qrot[r,r])
+    end
+    #puts "cnt = #{cnt}"
+    if self.row_size >= self.column_size
+      mu = self *  v * Matrix.diagonal(*s).inverse     
+      return [mu, v, s]
+    else
+      puts v.row_size
+      puts v.column_size
+      puts self.row_size
+      puts self.column_size
+      puts s.size
+
+      mu = (self.trans * v *  Matrix.diagonal(*s).inverse)
+      return [mu, v, s]
+    end
+  end
+  def []=(i,j,val)
+    @rows[i][j] = val
+  end
+end