otherinbox-classifier 1.3.1.1

data/Rakefile

@@ -0,0 +1,96 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rake/contrib/rubyforgepublisher'
+
+PKG_VERSION = "1.3.1"
+
+PKG_FILES = FileList[
+    "lib/**/*", "bin/*", "test/**/*", "[A-Z]*", "Rakefile", "html/**/*"
+]
+
+desc "Default Task"
+task :default => [ :test ]
+
+# Run the unit tests
+desc "Run all unit tests"
+Rake::TestTask.new("test") { |t|
+  t.libs << "lib"
+  t.pattern = 'test/*/*_test.rb'
+  t.verbose = true
+}
+
+# Make a console, useful when working on tests
+desc "Generate a test console"
+task :console do
+   verbose( false ) { sh "irb -I lib/ -r 'classifier'" }
+end
+
+# Genereate the RDoc documentation
+desc "Create documentation"
+Rake::RDocTask.new("doc") { |rdoc|
+  rdoc.title = "Ruby Classifier - Bayesian and LSI classification library"
+  rdoc.rdoc_dir = 'html'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+
+# Genereate the package
+spec = Gem::Specification.new do |s|
+
+  #### Basic information.
+
+  s.name = 'classifier'
+  s.version = PKG_VERSION
+  s.summary = <<-EOF
+   A general classifier module to allow Bayesian and other types of classifications.
+  EOF
+  s.description = <<-EOF
+   A general classifier module to allow Bayesian and other types of classifications.
+  EOF
+
+  #### Which files are to be included in this gem?  Everything!  (Except CVS directories.)
+
+  s.files = PKG_FILES
+
+  #### Load-time details: library and application (you will need one or both).
+
+  s.require_path = 'lib'
+  s.autorequire = 'classifier'
+
+  #### Documentation and testing.
+
+  s.has_rdoc = true
+
+  #### Dependencies and requirements.
+
+  s.add_dependency('stemmer', '>= 1.0.0')
+  s.requirements << "A porter-stemmer module to split word stems."
+
+  #### Author and project details.
+  s.author = "Lucas Carlson"
+  s.email = "lucas@rufy.com"
+  s.homepage = "http://classifier.rufy.com/"
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.need_zip = true
+  pkg.need_tar = true
+end
+
+desc "Report code statistics (KLOCs, etc) from the application"
+task :stats do
+  require 'code_statistics'
+  CodeStatistics.new(
+    ["Library", "lib"],
+    ["Units", "test"]
+  ).to_s
+end
+
+desc "Publish new documentation"
+task :publish do
+   `ssh rufy update-classifier-doc`
+    Rake::RubyForgePublisher.new('classifier', 'cardmagic').upload
+end

data/bin/bayes.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+begin
+	require 'rubygems'
+	require 'classifier'
+rescue
+	require 'classifier'
+end
+
+require 'madeleine'
+
+m = SnapshotMadeleine.new(File.expand_path("~/.bayes_data")) {
+	Classifier::Bayes.new 'Interesting', 'Uninteresting'
+}
+
+case ARGV[0]
+when "add"
+	case ARGV[1].downcase
+	when "interesting"
+		m.system.train_interesting File.open(ARGV[2]).read
+		puts "#{ARGV[2]} has been classified as interesting"
+	when "uninteresting"
+		m.system.train_uninteresting File.open(ARGV[2]).read
+		puts "#{ARGV[2]} has been classified as uninteresting"
+	else
+		puts "Invalid category: choose between interesting and uninteresting"
+		exit(1)
+	end
+when "classify"
+	puts m.system.classify(File.open(ARGV[1]).read)
+else
+	puts "Invalid option: choose add [category] [file] or clasify [file]"
+	exit(-1)
+end
+
+m.take_snapshot

data/bin/summarize.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+begin
+        require 'rubygems'
+        require 'classifier'
+rescue
+        require 'classifier'
+end
+
+require 'open-uri'
+
+num = ARGV[1].to_i
+num = num < 1 ? 10 : num
+
+text = open(ARGV.first).read
+puts text.gsub(/<[^>]+>/,"").gsub(/[\s]+/," ").summary(num)

data/lib/classifier.rb

@@ -0,0 +1,30 @@
+#--
+# Copyright (c) 2005 Lucas Carlson
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require 'rubygems'
+require 'classifier/extensions/string'
+require 'classifier/bayes'
+# require 'classifier/lsi'

data/lib/classifier/bayes.rb

@@ -0,0 +1,172 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+module Classifier
+
+class Bayes
+  # The class can be created with one or more categories, each of which will be
+  # initialized and given a training method. E.g., 
+  #      b = Classifier::Bayes.new 'Interesting', 'Uninteresting', 'Spam'
+	def initialize(*categories)
+		@categories = Hash.new
+		categories.each { |category| @categories[category.prepare_category_name] = Hash.new }
+		@total_words = 0
+	end
+
+	#
+	# Provides a general training method for all categories specified in Bayes#new
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train :this, "This text"
+	#     b.train "that", "That text"
+	#     b.train "The other", "The other text"
+	def train(category, text)
+		category = category.prepare_category_name
+		text.word_hash.each do |word, count|
+			@categories[category][word]     ||=     0
+			@categories[category][word]      +=     count
+			@total_words += count
+		end
+	end
+
+	#
+	# Provides a untraining method for all categories specified in Bayes#new
+	# Be very careful with this method.
+	#
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train :this, "This text"
+	#     b.untrain :this, "This text"
+	def untrain(category, text)
+		category = category.prepare_category_name
+		text.word_hash.each do |word, count|
+			if @total_words >= 0
+        # Sometimes items can be untrained before they are trained,
+        # be tolerant of that case
+        next if @categories[category][word].nil?
+				orig = @categories[category][word]
+				@categories[category][word]     ||=     0
+				@categories[category][word]      -=     count
+				if @categories[category][word] <= 0
+					@categories[category].delete(word)
+					count = orig
+				end
+				@total_words -= count
+			end
+		end
+	end
+		
+	#
+	# Returns the scores in each category the provided +text+. E.g.,
+	#    b.classifications "I hate bad words and you"
+	#    =>  {"Uninteresting"=>-12.6997928013932, "Interesting"=>-18.4206807439524}
+	# The largest of these scores (the one closest to 0) is the one picked out by #classify
+	def classifications(text)
+		score = Hash.new
+		@categories.each do |category, category_words|
+			score[category.to_s] = 0
+			total = category_words.values.inject(0) {|sum, element| sum+element}
+			text.word_hash.each do |word, count|
+				s = category_words.has_key?(word) ? category_words[word] : 0.1
+				score[category.to_s] += Math.log(s/total.to_f)
+			end
+		end
+		return score
+	end
+
+  # These assume that the classes are Member and Not Member
+  def myclassify(text)
+    myclassify_with_word_hash(text.word_hash)
+  end
+
+  # http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html
+  def myclassify_with_word_hash(word_hash)
+    member_term_count = @categories[:Member].size
+    nonmember_term_count = @categories[:"Not member"].size
+    term_count = member_term_count + nonmember_term_count
+    score = 0
+    word_hash.each do |word, count|
+      # count of words in each category
+      member_count = @categories[:Member][word].to_i + 1
+      nonmember_count = @categories[:"Not member"][word].to_i + 1
+      next if member_count.to_i == 1 && nonmember_count.to_i == 1      
+
+      # find relative prob word is in class -- p(w|c)
+      word_member_p = (member_count) / (total_member_count + term_count).to_f
+      word_nonmember_p =  (nonmember_count) / (total_nonmember_count + term_count).to_f
+      
+      word_pr = Math.log(word_member_p / word_nonmember_p)
+      score += word_pr * count
+    end 
+    if score > 0
+      return "Member", score
+    else
+      return "Not member", score
+    end
+  end
+
+  #
+  # Returns the classification of the provided +text+, which is one of the 
+  # categories given in the initializer. E.g.,
+  #    b.classify "I hate bad words and you"
+  #    =>  'Uninteresting'
+	def classify(text)
+		(classifications(text).sort_by { |a| -a[1] })[0][0]
+	end
+	
+	#
+	# Provides training and untraining methods for the categories specified in Bayes#new
+	# For example:
+	#     b = Classifier::Bayes.new 'This', 'That', 'the_other'
+	#     b.train_this "This text"
+	#     b.train_that "That text"
+	#     b.untrain_that "That text"
+	#     b.train_the_other "The other text"
+	def method_missing(name, *args)
+		category = name.to_s.gsub(/(un)?train_([\w]+)/, '\2').prepare_category_name
+		if @categories.has_key? category
+			args.each { |text| eval("#{$1}train(category, text)") }
+		elsif name.to_s =~ /(un)?train_([\w]+)/
+			raise StandardError, "No such category: #{category}"
+		else
+	    super  #raise StandardError, "No such method: #{name}"
+		end
+	end
+	
+	#
+	# Provides a list of category names
+	# For example:
+	#     b.categories
+	#     =>   ['This', 'That', 'the_other']
+	def categories # :nodoc:
+		@categories.keys.collect {|c| c.to_s}
+	end
+	
+	#
+	# Allows you to add categories to the classifier.
+	# For example:
+	#     b.add_category "Not spam"
+	#
+	# WARNING: Adding categories to a trained classifier will
+	# result in an undertrained category that will tend to match
+	# more criteria than the trained selective categories. In short,
+	# try to initialize your categories at initialization.
+	def add_category(category)
+		@categories[category.prepare_category_name] = Hash.new
+	end
+	
+	alias append_category add_category
+
+  private
+  def total_member_count
+    @total_member_count ||= @categories[:Member].values.inject(0) {|sum, element| sum+element}
+  end
+
+  def total_nonmember_count
+    @total_nonmember_count ||= @categories[:"Not member"].values.inject(0) {|sum, element| sum+element}
+  end
+
+end
+
+end

data/lib/classifier/extensions/string.rb

@@ -0,0 +1,16 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+begin
+	require 'stemmer'
+rescue LoadError
+	puts "Please install stemmer from http://rubyforge.org/projects/stemmer or 'gem install stemmer'"
+	exit(-1)
+end
+
+require 'classifier/extensions/word_hash'
+
+class Object
+	def prepare_category_name; to_s.gsub("_"," ").capitalize.intern end
+end

data/lib/classifier/extensions/vector.rb

@@ -0,0 +1,106 @@
+# Author::    Ernest Ellingson
+# Copyright:: Copyright (c) 2005 
+
+# These are extensions to the std-lib 'matrix' to allow an all ruby SVD
+
+require 'matrix'
+require 'mathn'
+
+class Array
+   def sum
+      inject(0) { |sum,term| sum += term  }.to_f
+   end
+end
+
+class Vector
+  def magnitude
+    sumsqs = 0.0
+    self.size.times do |i|
+      sumsqs += self[i] ** 2.0 
+    end
+    Math.sqrt(sumsqs)
+  end
+  def normalize
+    nv = []
+    mag = self.magnitude
+    self.size.times do |i|
+
+      nv << (self[i] / mag)
+
+    end
+    Vector[*nv]
+  end
+end
+
+class Matrix
+  def Matrix.diag(s)
+     Matrix.diagonal(*s)
+  end
+  
+  alias :trans :transpose
+
+  def SV_decomp(maxSweeps = 20)
+    if self.row_size >= self.column_size
+      q = self.trans * self
+    else
+      q = self * self.trans
+    end
+    
+    qrot    = q.dup
+    v       = Matrix.identity(q.row_size)
+    azrot   = nil
+    mzrot   = nil
+    cnt     = 0
+    s_old   = nil
+    mu      = nil
+
+    while true do
+      cnt += 1
+      for row in (0...qrot.row_size-1) do
+        for col in (1..qrot.row_size-1) do
+          next if row == col
+          h = Math.atan((2 * qrot[row,col])/(qrot[row,row]-qrot[col,col]))/2.0
+          hcos = Math.cos(h)
+          hsin = Math.sin(h)
+          mzrot = Matrix.identity(qrot.row_size)
+          mzrot[row,row] = hcos
+          mzrot[row,col] = -hsin
+          mzrot[col,row] = hsin
+          mzrot[col,col] = hcos
+          qrot = mzrot.trans * qrot * mzrot
+          v = v * mzrot
+        end 
+      end
+      s_old = qrot.dup if cnt == 1
+      sum_qrot = 0.0 
+      if cnt > 1
+        qrot.row_size.times do |r|
+          sum_qrot += (qrot[r,r]-s_old[r,r]).abs if (qrot[r,r]-s_old[r,r]).abs > 0.001
+        end
+        s_old = qrot.dup
+      end 
+      break if (sum_qrot <= 0.001 and cnt > 1) or cnt >= maxSweeps
+    end # of do while true
+    s = []
+    qrot.row_size.times do |r|
+      s << Math.sqrt(qrot[r,r])
+    end
+    #puts "cnt = #{cnt}"
+    if self.row_size >= self.column_size
+      mu = self *  v * Matrix.diagonal(*s).inverse     
+      return [mu, v, s]
+    else
+      puts v.row_size
+      puts v.column_size
+      puts self.row_size
+      puts self.column_size
+      puts s.size
+
+      mu = (self.trans * v *  Matrix.diagonal(*s).inverse)
+      return [mu, v, s]
+    end
+  end
+  def []=(i,j,val)
+    @rows[i][j] = val
+  end
+end