classifier 1.0

data/README

@@ -0,0 +1,33 @@
+== Welcome to Classifier
+
+Classifier is a general module to allow Bayesian and other types of classifications.
+
+== Usage
+    require 'classifier'
+    b = Classifier::Bayes.new 'Interesting', 'Uninteresting'
+    b.train_interesting "here are some good words. I hope you love them"
+    b.train_uninteresting "here are some bad words, I hate you"
+    b.classify "I hate bad words and you" # returns 'Uninsteresting'
+    
+    require 'madeleine'
+    m = SnapshotMadeleine.new("bayes_data") {
+        Classifier::Bayes.new 'Interesting', 'Uninteresting'
+    }
+    m.system.train_interesting "here are some good words. I hope you love them"
+    m.system.train_uninteresting "here are some bad words, I hate you"
+    m.take_snapshot
+    m.system.classify "I love you" # returns 'Interesting'
+
+Using Madeleine, your application can persist the learned data over time.
+
+== Bayesian Classification
+
+* http://www.process.com/precisemail/bayesian_filtering.htm
+* http://en.wikipedia.org/wiki/Bayesian_filtering
+* http://www.paulgraham.com/spam.html
+
+== About
+
+Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+Copyright:: Copyright (c) 2005 Lucas Carlson
+License::   GPL

data/Rakefile

@@ -0,0 +1,77 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rake/contrib/rubyforgepublisher'
+
+PKG_VERSION = "1.0.0"
+
+PKG_FILES = FileList[
+    "lib/**/*", "bin/*", "test/**/*", "[A-Z]*", "Rakefile"
+]
+
+desc "Default Task"
+task :default => [ :test ]
+
+# Run the unit tests
+desc "Run all unit tests"
+Rake::TestTask.new("test") { |t|
+  t.libs << "lib"
+  t.pattern = 'test/*/*_test.rb'
+  t.verbose = true
+}
+
+# Genereate the RDoc documentation
+desc "Create documentation"
+Rake::RDocTask.new("doc") { |rdoc|
+  rdoc.rdoc_dir = 'doc'
+  rdoc.title    = "Classifier library"
+  rdoc.options << '--line-numbers --inline-source --accessor'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+
+# Genereate the package
+spec = Gem::Specification.new do |s|
+
+  #### Basic information.
+
+  s.name = 'classifier'
+  s.version = "1.0"
+  s.summary = <<-EOF
+   A general classifier module to allow Bayesian and other types of classifications.
+  EOF
+  s.description = <<-EOF
+   A general classifier module to allow Bayesian and other types of classifications.
+  EOF
+
+  #### Which files are to be included in this gem?  Everything!  (Except CVS directories.)
+
+  s.files = PKG_FILES
+
+  #### Load-time details: library and application (you will need one or both).
+
+  s.require_path = 'lib'
+  s.autorequire = 'classifier'
+
+  #### Documentation and testing.
+
+  s.has_rdoc = true
+
+  #### Author and project details.
+
+  s.author = "Lucas Carlson"
+  s.email = "lucas@rufy.com"
+  s.homepage = "http://rubyforge.org/projects/classifier/"
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.need_zip = false
+  pkg.need_tar = true
+end
+
+desc "Publish to RubyForge"
+task :rubyforge do
+    Rake::RubyForgePublisher.new('classifier', 'cardmagic').upload
+end

data/bin/bayes.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'classifier'
+require 'madeleine'
+
+m = SnapshotMadeleine.new(File.expand_path("~/.bayes_data")) {
+	Classifier::Bayes.new 'Interesting', 'Uninteresting'
+}
+
+case ARGV[0]
+when "add"
+	case ARGV[1].downcase
+	when "interesting"
+		m.system.train_interesting File.open(ARGV[2]).read
+		puts "#{ARGV[2]} has been classified as interesting"
+	when "uninteresting"
+		m.system.train_uninteresting File.open(ARGV[2]).read
+		puts "#{ARGV[2]} has been classified as uninteresting"
+	else
+		puts "Invalid category: choose between interesting and uninteresting"
+		exit(1)
+	end
+when "classify"
+	puts m.system.classify(File.open(ARGV[1]).read)
+else
+	puts "Invalid option: choose add [category] [file] or clasify [file]"
+	exit(1)
+end
+
+m.take_snapshot

data/lib/classifier.rb

@@ -0,0 +1,28 @@
+#--
+# Copyright (c) 2005 Lucas Carlson
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require 'classifier/string_extensions'
+require 'classifier/bayes'

data/lib/classifier/bayes.rb

@@ -0,0 +1,53 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+module Classifier
+
+class Bayes
+	def initialize(*categories)
+		@categories = Hash.new
+		categories.each { |category| @categories[category.capitalize.intern] = Hash.new }
+		@total_words = 0
+	end
+
+	def classify(text)
+		(classifications(text).sort { |a, b| b[1] <=> a[1] })[0][0]
+	end
+	
+	def classifications(text)
+		score = Hash.new
+		@categories.each do |category, category_words|
+			score[category.to_s] = 0
+			total = category_words.values.inject(0) {|sum, element| sum+element}
+			text.word_hash.each do |word, count|
+				s = category_words.has_key?(word) ? category_words[word] : 0.1
+				score[category.to_s] += Math.log(s/total.to_f)
+			end
+		end
+		return score
+	end
+
+	def method_missing(name, *args)
+		category = name.to_s.gsub(/train_([\w]+)/, '\1').capitalize.intern
+		if @categories.has_key? category
+			args.each {|text| add_words category, text}
+		elsif name.to_s =~ /train_([\w]+)/
+			raise StandardError, "No such category: #{category}"
+		else
+			raise StandardError, "No such method: #{name}"
+		end
+	end
+
+	private
+
+	def add_words(category, text)
+		text.word_hash.each do |word, count|
+			@categories[category][word]	||=	0
+			@categories[category][word]	 +=	count
+			@total_words += count
+		end
+	end
+end
+
+end

data/lib/classifier/string_extensions.rb

@@ -0,0 +1,11 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require 'classifier/string_extensions/porter_stemmer'
+require 'classifier/string_extensions/word_hash'
+
+class String
+	include Classifier::Stemmable
+	include Classifier::WordHash
+end

data/lib/classifier/string_extensions/porter_stemmer.rb

@@ -0,0 +1,188 @@
+module Classifier
+	
+module Stemmable
+
+  STEP_2_LIST = {
+    'ational'=>'ate', 'tional'=>'tion', 'enci'=>'ence', 'anci'=>'ance',
+    'izer'=>'ize', 'bli'=>'ble',
+    'alli'=>'al', 'entli'=>'ent', 'eli'=>'e', 'ousli'=>'ous',
+    'ization'=>'ize', 'ation'=>'ate',
+    'ator'=>'ate', 'alism'=>'al', 'iveness'=>'ive', 'fulness'=>'ful',
+    'ousness'=>'ous', 'aliti'=>'al',
+    'iviti'=>'ive', 'biliti'=>'ble', 'logi'=>'log'
+  }
+  
+  STEP_3_LIST = {
+    'icate'=>'ic', 'ative'=>'', 'alize'=>'al', 'iciti'=>'ic',
+    'ical'=>'ic', 'ful'=>'', 'ness'=>''
+  }
+
+
+  SUFFIX_1_REGEXP = /(
+                    ational  |
+                    tional   |
+                    enci     |
+                    anci     |
+                    izer     |
+                    bli      |
+                    alli     |
+                    entli    |
+                    eli      |
+                    ousli    |
+                    ization  |
+                    ation    |
+                    ator     |
+                    alism    |
+                    iveness  |
+                    fulness  |
+                    ousness  |
+                    aliti    |
+                    iviti    |
+                    biliti   |
+                    logi)$/x
+
+
+  SUFFIX_2_REGEXP = /(
+                      al       |
+                      ance     |
+                      ence     |
+                      er       |
+                      ic       | 
+                      able     |
+                      ible     |
+                      ant      |
+                      ement    |
+                      ment     |
+                      ent      |
+                      ou       |
+                      ism      |
+                      ate      |
+                      iti      |
+                      ous      |
+                      ive      |
+                      ize)$/x
+
+
+  C = "[^aeiou]"         # consonant
+  V = "[aeiouy]"         # vowel
+  CC = "#{C}(?>[^aeiouy]*)"  # consonant sequence
+  VV = "#{V}(?>[aeiou]*)"    # vowel sequence
+
+  MGR0 = /^(#{CC})?#{VV}#{CC}/o                # [cc]vvcc... is m>0
+  MEQ1 = /^(#{CC})?#{VV}#{CC}(#{VV})?$/o       # [cc]vvcc[vv] is m=1
+  MGR1 = /^(#{CC})?#{VV}#{CC}#{VV}#{CC}/o      # [cc]vvccvvcc... is m>1
+  VOWEL_IN_STEM   = /^(#{CC})?#{V}/o                      # vowel in stem
+
+  #
+  # Porter stemmer in Ruby.
+  #
+  # This is the Porter stemming algorithm, ported to Ruby from the
+  # version coded up in Perl.  It's easy to follow against the rules
+  # in the original paper in:
+  #
+  #   Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
+  #   no. 3, pp 130-137,
+  #
+  # See also http://www.tartarus.org/~martin/PorterStemmer
+  #
+  # Send comments to raypereda@hotmail.com
+  #
+  
+  def stem_porter
+
+    # make a copy of the given object and convert it to a string.
+    w = self.dup.to_str
+    
+    return w if w.length < 3
+    
+    # now map initial y to Y so that the patterns never treat it as vowel
+    w[0] = 'Y' if w[0] == ?y
+    
+    # Step 1a
+    if w =~ /(ss|i)es$/
+      w = $` + $1
+    elsif w =~ /([^s])s$/ 
+      w = $` + $1
+    end
+
+    # Step 1b
+    if w =~ /eed$/
+      w.chop! if $` =~ MGR0 
+    elsif w =~ /(ed|ing)$/
+      stem = $`
+      if stem =~ VOWEL_IN_STEM 
+        w = stem
+	case w
+        when /(at|bl|iz)$/             then w << "e"
+        when /([^aeiouylsz])\1$/       then w.chop!
+        when /^#{CC}#{V}[^aeiouwxy]$/o then w << "e"
+        end
+      end
+    end
+
+    if w =~ /y$/ 
+      stem = $`
+      w = stem + "i" if stem =~ VOWEL_IN_STEM 
+    end
+
+    # Step 2
+    if w =~ SUFFIX_1_REGEXP
+      stem = $`
+      suffix = $1
+      # print "stem= " + stem + "\n" + "suffix=" + suffix + "\n"
+      if stem =~ MGR0
+        w = stem + STEP_2_LIST[suffix]
+      end
+    end
+
+    # Step 3
+    if w =~ /(icate|ative|alize|iciti|ical|ful|ness)$/
+      stem = $`
+      suffix = $1
+      if stem =~ MGR0
+        w = stem + STEP_3_LIST[suffix]
+      end
+    end
+
+    # Step 4
+    if w =~ SUFFIX_2_REGEXP
+      stem = $`
+      if stem =~ MGR1
+        w = stem
+      end
+    elsif w =~ /(s|t)(ion)$/
+      stem = $` + $1
+      if stem =~ MGR1
+        w = stem
+      end
+    end
+
+    #  Step 5
+    if w =~ /e$/ 
+      stem = $`
+      if (stem =~ MGR1) ||
+          (stem =~ MEQ1 && stem !~ /^#{CC}#{V}[^aeiouwxy]$/o)
+        w = stem
+      end
+    end
+
+    if w =~ /ll$/ && w =~ MGR1
+      w.chop!
+    end
+
+    # and turn initial Y back to y
+    w[0] = 'y' if w[0] == ?Y
+
+    w
+  end
+
+
+  #
+  # make the stem_porter the default stem method, just in case we
+  # feel like having multiple stemmers available later.
+  #
+  alias stem stem_porter
+
+end
+
+end

data/lib/classifier/string_extensions/word_hash.rb

@@ -0,0 +1,24 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+module Classifier
+
+module WordHash  
+  def without_punctuation
+    tr( ',?.!;:\'"@#$%^&*()_=+[]{}\|<>/`~', " " )
+  end
+  
+	def word_hash
+		d = Hash.new
+		corpus = without_punctuation
+		(corpus.split + gsub(/[\w]/,"").split).each do |word|
+			key = word.downcase.stem.intern
+			d[key] ||= 0
+			d[key] += 1
+		end
+		return d
+	end
+end
+
+end

data/test/bayes/bayesian_test.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class BayesianTest < Test::Unit::TestCase
+	def setup
+		@classifier = Classifier::Bayes.new 'Interesting', 'Uninteresting'
+	end
+	
+	def test_good_training
+		assert_nothing_raised { @classifier.train_interesting "love" }
+	end
+
+	def test_bad_training
+		assert_raise(StandardError) { @classifier.train_no_category "words" }
+	end
+
+	def test_classification
+		@classifier.train_interesting "here are some good words. I hope you love them"
+		@classifier.train_uninteresting "here are some bad words, I hate you"
+		assert_equal 'Uninteresting', @classifier.classify("I hate bad words and you")
+	end
+end

data/test/bayes/string_extensions_test.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class StringExtensionsTest < Test::Unit::TestCase
+	def test_word_hash
+		hash = {:some=>1, :good=>1, :hope=>1, :word=>1, :you=>1, :here=>1, :love=>1, :i=>1, :ar=>1, :them=>1, :"."=>1, :"!"=>1}
+		
+		assert_equal hash, "here are some good words. I hope you love them!".word_hash
+	end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'test/unit'
+require 'classifier'

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.6
+specification_version: 1
+name: classifier
+version: !ruby/object:Gem::Version 
+  version: "1.0"
+date: 2005-04-10
+summary: A general classifier module to allow Bayesian and other types of classifications.
+require_paths: 
+  - lib
+email: lucas@rufy.com
+homepage: http://rubyforge.org/projects/classifier/
+rubyforge_project: 
+description: A general classifier module to allow Bayesian and other types of classifications.
+autorequire: classifier
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+  - Lucas Carlson
+files: 
+  - lib/classifier
+  - lib/classifier.rb
+  - lib/classifier/bayes.rb
+  - lib/classifier/string_extensions
+  - lib/classifier/string_extensions.rb
+  - lib/classifier/string_extensions/porter_stemmer.rb
+  - lib/classifier/string_extensions/word_hash.rb
+  - bin/bayes.rb
+  - test/bayes
+  - test/test_helper.rb
+  - test/bayes/bayesian_test.rb
+  - test/bayes/string_extensions_test.rb
+  - Rakefile
+  - README
+test_files: []
+rdoc_options: []
+extra_rdoc_files: []
+executables: []
+extensions: []
+requirements: []
+dependencies: []