classifier 1.1 → 1.1.1

data/lib/classifier/bayes.rb

@@ -5,16 +5,36 @@
 module Classifier
 
 class Bayes
+  # The class can be created with one or more categories, each of which will be
+  # initialized and given a training method. E.g., 
+  #      b = Classifier::Bayes.new 'Interesting', 'Uninteresting', 'Spam'
 	def initialize(*categories)
 		@categories = Hash.new
-		categories.each { |category| @categories[category.capitalize.intern] = Hash.new }
+		categories.each { |category| @categories[category.to_s.gsub("_"," ").capitalize.intern] = Hash.new }
 		@total_words = 0
 	end
 
-	def classify(text)
-		(classifications(text).sort_by { |a| -a[1] })[0][0]
+	#
+	# Provides a general training method for all categories specified in Bayes#new
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train :this, "This text"
+	#     b.train "that", "That text"
+	#     b.train "The other", "The other text"
+	def train(category, text)
+		category = category.to_s.gsub("_"," ").capitalize.intern
+		text.word_hash.each do |word, count|
+			@categories[category][word]     ||=     0
+			@categories[category][word]      +=     count
+			@total_words += count
+		end
 	end
 	
+	#
+	# Returns the scores in each category the provided +text+. E.g.,
+	#    b.classifications "I hate bad words and you"
+	#    =>  {"Uninteresting"=>-12.6997928013932, "Interesting"=>-18.4206807439524}
+	# The largest of these scores (the one closest to 0) is the one picked out by #classify
 	def classifications(text)
 		score = Hash.new
 		@categories.each do |category, category_words|
@@ -28,6 +48,22 @@ class Bayes
 		return score
 	end
 
+  #
+  # Returns the classification of the provided +text+, which is one of the 
+  # categories given in the initializer. E.g.,
+  #    b.classify "I hate bad words and you"
+  #    =>  'Uninteresting'
+	def classify(text)
+		(classifications(text).sort_by { |a| -a[1] })[0][0]
+	end
+	
+	#
+	# Provides training methods for the categories specified in Bayes#new
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train_this "This text"
+	#     b.train_that "That text"
+	#     b.train_the_other "The other text"
 	def method_missing(name, *args)
 		category = name.to_s.gsub(/train_([\w]+)/, '\1').gsub("_"," ").capitalize.intern
 		if @categories.has_key? category
@@ -35,18 +71,33 @@ class Bayes
 		elsif name.to_s =~ /train_([\w]+)/
 			raise StandardError, "No such category: #{category}"
 		else
-			raise StandardError, "No such method: #{name}"
+	    super  #raise StandardError, "No such method: #{name}"
 		end
 	end
-
-	def train(category, text)
-		category = category.to_s.gsub("_"," ").capitalize.intern
-		text.word_hash.each do |word, count|
-			@categories[category][word]	||=	0
-			@categories[category][word]	 +=	count
-			@total_words += count
-		end
+	
+	#
+	# Provides a list of category names
+	# For example:
+	#     b.categories
+	#     =>   ['This', 'That', 'the_other']
+	def categories # :nodoc:
+		@categories.keys.collect {|c| c.to_s}
 	end
+	
+	#
+	# Allows you to add categories to the classifier.
+	# For example:
+	#     b.add_category "Not spam"
+	#
+	# WARNING: Adding categories to a trained classifier will
+	# result in an undertrained category that will tend to match
+	# more criteria than the trained selective categories. In short,
+	# try to initialize your categories at initialization.
+	def add_category(category)
+		@categories[category.to_s.gsub("_"," ").capitalize.intern] = Hash.new
+	end
+	
+	alias append_category add_category # :nodoc:
 end
 
 end

data/lib/classifier/string_extensions/porter_stemmer.rb

@@ -7,7 +7,21 @@
 # Copyright 2005 Greg Fast <gdf@speakeasy.net>
 
 module Classifier
-	
+
+#
+# Porter stemmer in Ruby.
+#
+# This is the Porter stemming algorithm, ported to Ruby from the
+# version coded up in Perl.  It's easy to follow against the rules
+# in the original paper in:
+#
+#   Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
+#   no. 3, pp 130-137,
+#
+# See also http://www.tartarus.org/~martin/PorterStemmer
+#
+# Send comments to raypereda@hotmail.com
+#
 module Stemmable
 
   STEP_2_LIST = {
@@ -82,20 +96,9 @@ module Stemmable
   VOWEL_IN_STEM   = /^(#{CC})?#{V}/o                      # vowel in stem
 
   #
-  # Porter stemmer in Ruby.
-  #
-  # This is the Porter stemming algorithm, ported to Ruby from the
-  # version coded up in Perl.  It's easy to follow against the rules
-  # in the original paper in:
-  #
-  #   Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
-  #   no. 3, pp 130-137,
-  #
-  # See also http://www.tartarus.org/~martin/PorterStemmer
-  #
-  # Send comments to raypereda@hotmail.com
-  #
-  
+  # Stems the word contained in the current object. E.g.,
+  #     "actually".stem_porter
+  #     => "actual"
   def stem_porter
 
     # make a copy of the given object and convert it to a string.

data/lib/classifier/string_extensions/word_hash.rb

@@ -4,23 +4,116 @@
 
 module Classifier
 
+# This module is mixed into String to provide convenience 
+# methods for the Classifier package.
 module WordHash  
+  
+  # Removes common punctuation symbols, returning a new string. E.g.,
+  #     "Hello (greeting's), with {braces} < >...?".without_punctuation
+  #     => "Hello  greetings   with  braces         "
   def without_punctuation
-    tr( ',?.!;:\'"@#$%^&*()_=+[]{}\|<>/`~', " " )
+    tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
   end
   
+  # Return a Hash of strings => ints. Each word in the string is stemmed,
+  # interned, and indexes to its frequency in the document.  
 	def word_hash
 		d = Hash.new
 		corpus = without_punctuation
 		(corpus.split + gsub(/[\w+]/,"").split).each do |word|
-			key = word.downcase.stem.intern
+			item = word.downcase
+			key = item.stem.intern
 			if !(word =~ /[\w+]/) || word.length > 2
 				d[key] ||= 0
 				d[key] += 1
-			end
+			end unless CORPUS_SKIP_WORDS[item]
 		end
 		return d
 	end
+	
+	private 
+	CORPUS_SKIP_WORDS = {
+      "a" => 1,
+      "again" => 1,
+      "all" => 1,
+      "along" => 1,
+      "are" => 1,
+      "also" => 1,
+      "an" => 1,
+      "and" => 1,
+      "as" => 1,
+      "at" => 1,
+      "but" => 1,
+      "by" => 1,
+      "came" => 1,
+      "can" => 1,
+      "cant" => 1,
+      "couldnt" => 1,
+      "did" => 1,
+      "didn" => 1,
+      "didnt" => 1,
+      "do" => 1,
+      "doesnt" => 1,
+      "dont" => 1,
+      "ever" => 1,
+      "first" => 1,
+      "from" => 1,
+      "have" => 1,
+      "her" => 1,
+      "here" => 1,
+      "him" => 1,
+      "how" => 1,
+      "i" => 1,
+      "if" => 1,
+      "in" => 1,
+      "into" => 1,
+      "is" => 1,
+      "isnt" => 1,
+      "it" => 1,
+      "itll" => 1,
+      "just" => 1,
+      "last" => 1,
+      "least" => 1,
+      "like" => 1,
+      "most" => 1,
+      "my" => 1,
+      "new" => 1,
+      "no" => 1,
+      "not" => 1,
+      "now" => 1,
+      "of" => 1,
+      "on" => 1,
+      "or" => 1,
+      "should" => 1,
+      "sinc" => 1,
+      "so" => 1,
+      "some" => 1,
+      "th" => 1,
+      "than" => 1,
+      "this" => 1,
+      "that" => 1,
+      "the" => 1,
+      "their" => 1,
+      "then" => 1,
+      "those" => 1,
+      "to" => 1,
+      "told" => 1,
+      "too" => 1,
+      "true" => 1,
+      "try" => 1,
+      "until" => 1,
+      "url" => 1,
+      "us" => 1,
+      "were" => 1,
+      "when" => 1,
+      "whether" => 1,
+      "while" => 1,
+      "with" => 1,
+      "within" => 1,
+      "yes" => 1,
+      "you" => 1,
+      "youll" => 1,
+      }
 end
 
 end

data/test/bayes/bayesian_test.rb

@@ -11,6 +11,19 @@ class BayesianTest < Test::Unit::TestCase
 	def test_bad_training
 		assert_raise(StandardError) { @classifier.train_no_category "words" }
 	end
+	
+	def test_bad_method
+		assert_raise(NoMethodError) { @classifier.forget_everything_you_know "" }
+	end
+	
+	def test_categories
+		assert_equal ['Interesting', 'Uninteresting'], @classifier.categories
+	end
+
+	def test_add_category
+		@classifier.add_category 'Test'
+		assert_equal ['Interesting', 'Test', 'Uninteresting'], @classifier.categories
+	end
 
 	def test_classification
 		@classifier.train_interesting "here are some good words. I hope you love them"

data/test/string_extensions/word_hash_test.rb

@@ -1,8 +1,12 @@
 require File.dirname(__FILE__) + '/../test_helper'
 class StringExtensionsTest < Test::Unit::TestCase
 	def test_word_hash
-		hash = {:some=>1, :good=>1, :hope=>1, :word=>1, :you=>1, :here=>1, :love=>1, :ar=>1, :them=>1, :"."=>1, :"!"=>1}
-		
-		assert_equal hash, "here are some good words. I hope you love them!".word_hash
+		hash = {:good=>1, :"!"=>1, :hope=>1, :"'."=>1, :love=>1, :word=>1, :them=>1, :test=>1}
+		assert_equal hash, "here are some good words of test's. I hope you love them!".word_hash
+	end
+	
+	def test_without_punctuation
+	  sample = "Hello  thats welcome    to   no    punctuation"
+	  assert_equal sample, "Hello, that's welcome () to ! no *& punctuation".without_punctuation
 	end
 end

metadata

@@ -3,7 +3,7 @@ rubygems_version: 0.8.6
 specification_version: 1
 name: classifier
 version: !ruby/object:Gem::Version 
-  version: "1.1"
+  version: 1.1.1
 date: 2005-04-11
 summary: A general classifier module to allow Bayesian and other types of classifications.
 require_paths: 
@@ -42,6 +42,41 @@ files:
   - test/string_extensions/word_hash_test.rb
   - Rakefile
   - README
+  - doc/classes
+  - doc/created.rid
+  - doc/files
+  - doc/fr_class_index.html
+  - doc/fr_file_index.html
+  - doc/fr_method_index.html
+  - doc/index.html
+  - doc/rdoc-style.css
+  - doc/classes/Classifier
+  - doc/classes/Classifier.html
+  - doc/classes/String.html
+  - doc/classes/Classifier/Bayes.html
+  - doc/classes/Classifier/Bayes.src
+  - doc/classes/Classifier/Stemmable.html
+  - doc/classes/Classifier/Stemmable.src
+  - doc/classes/Classifier/WordHash.html
+  - doc/classes/Classifier/WordHash.src
+  - doc/classes/Classifier/Bayes.src/M000005.html
+  - doc/classes/Classifier/Bayes.src/M000006.html
+  - doc/classes/Classifier/Bayes.src/M000007.html
+  - doc/classes/Classifier/Bayes.src/M000008.html
+  - doc/classes/Classifier/Bayes.src/M000009.html
+  - doc/classes/Classifier/Bayes.src/M000010.html
+  - doc/classes/Classifier/Stemmable.src/M000003.html
+  - doc/classes/Classifier/WordHash.src/M000001.html
+  - doc/classes/Classifier/WordHash.src/M000002.html
+  - doc/files/lib
+  - doc/files/README.html
+  - doc/files/lib/classifier
+  - doc/files/lib/classifier_rb.html
+  - doc/files/lib/classifier/bayes_rb.html
+  - doc/files/lib/classifier/string_extensions
+  - doc/files/lib/classifier/string_extensions_rb.html
+  - doc/files/lib/classifier/string_extensions/porter_stemmer_rb.html
+  - doc/files/lib/classifier/string_extensions/word_hash_rb.html
 test_files: []
 rdoc_options: []
 extra_rdoc_files: []