logankoester-classifier 1.4.3

data/lib/classifier/stopwords/ru

@@ -0,0 +1,161 @@
+# Russian stopwords
+# http://snowball.tartarus.org/algorithms/russian/stop.txt
+и              # and
+в              # in/into
+во             # alternative form
+не             # not
+что            # what/that
+он             # he
+на             # on/onto
+я              # i
+с              # from
+со             # alternative form
+как            # how
+а              # milder form of `no' (but)
+то             # conjunction and form of `that'
+все            # all
+она            # she
+так            # so, thus
+его            # him
+но             # but
+да             # yes/and
+ты             # thou
+к              # towards, by
+у              # around, chez
+же             # intensifier particle
+вы             # you
+за             # beyond, behind
+бы             # conditional/subj. particle
+по             # up to, along
+только         # only
+ее             # her
+мне            # to me
+было           # it was
+вот            # here is/are, particle
+от             # away from
+меня           # me
+еще            # still, yet, more
+нет            # no, there isnt/arent
+о              # about
+из             # out of
+ему            # to him
+теперь         # now
+когда          # when
+даже           # even
+ну             # so, well
+вдруг          # suddenly
+ли             # interrogative particle
+если           # if
+уже            # already, but homonym of `narrower'
+или            # or
+ни             # neither
+быть           # to be
+был            # he was
+него           # prepositional form of его
+до             # up to
+вас            # you accusative
+нибудь         # indef. suffix preceded by hyphen
+опять          # again
+уж             # already, but homonym of `adder'
+вам            # to you
+сказал         # he said
+ведь           # particle `after all'
+там            # there
+потом          # then
+себя           # oneself
+ничего         # nothing
+ей             # to her
+может          # usually with `быть' as `maybe'
+они            # they
+тут            # here
+где            # where
+есть           # there is/are
+надо           # got to, must
+ней            # prepositional form of  ей
+для            # for
+мы             # we
+тебя           # thee
+их             # them, their
+чем            # than
+была           # she was
+сам            # self
+чтоб           # in order to
+без            # without
+будто          # as if
+человек        # man, person, one
+чего           # genitive form of `what'
+раз            # once
+тоже           # also
+себе           # to oneself
+под            # beneath
+жизнь          # life
+будет          # will be
+ж              # short form of intensifer particle `же'
+тогда          # then
+кто            # who
+этот           # this
+говорил        # was saying
+того           # genitive form of `that'
+потому         # for that reason
+этого          # genitive form of `this'
+какой          # which
+совсем         # altogether
+ним            # prepositional form of `его', `они'
+здесь          # here
+этом           # prepositional form of `этот'
+один           # one
+почти          # almost
+мой            # my
+тем            # instrumental/dative plural of `тот', `то'
+чтобы          # full form of `in order that'
+нее            # her (acc.)
+кажется        # it seems
+сейчас         # now
+были           # they were
+куда           # where to
+зачем          # why
+сказать        # to say
+всех           # all (acc., gen. preposn. plural)
+никогда        # never
+сегодня        # today
+можно          # possible, one can
+при            # by
+наконец        # finally
+два            # two
+об             # alternative form of `о', about
+другой         # another
+хоть           # even
+после          # after
+над            # above
+больше         # more
+тот            # that one (masc.)
+через          # across, in
+эти            # these
+нас            # us
+про            # about
+всего          # in all, only, of all
+них            # prepositional form of `они' (they)
+какая          # which, feminine
+много          # lots
+разве          # interrogative particle
+сказала        # she said
+три            # three
+эту            # this, acc. fem. sing.
+моя            # my, feminine
+впрочем        # moreover, besides
+хорошо         # good
+свою           # ones own, acc. fem. sing.
+этой           # oblique form of `эта', fem. `this'
+перед          # in front of
+иногда         # sometimes
+лучше          # better
+чуть           # a little
+том            # preposn. form of `that one'
+нельзя         # one must not
+такой          # such a one
+им             # to them
+более          # more
+всегда         # always
+конечно        # of course
+всю            # acc. fem. sing of `all'
+между          # between

data/lib/init.rb

@@ -0,0 +1 @@
+require 'classifier'

data/tasks/test.rake

@@ -0,0 +1,6 @@
+require 'rake/testtask'
+ 
+ 
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/**/*_test.rb']
+end

data/test/base_test.rb

@@ -0,0 +1,17 @@
+require File.dirname(__FILE__) + '/test_helper'
+class HelpersTest < Test::Unit::TestCase
+  
+	def test_word_hash
+	  c = Classifier::Base.new
+		hash = {'good'=>1, "!"=>1, 'hope'=>1, "'"=>1, "."=>1, 'love'=>1, 'word'=>1, 'them'=>1, 'test'=>1}
+		assert_equal hash, c.word_hash("here are some good words of test's. I hope you love them!")
+	end
+
+   	
+	def test_clean_word_hash
+	  c = Classifier::Base.new
+	  hash = {'good'=>1, 'word'=>1, 'hope'=>1, 'love'=>1, 'them'=>1, 'test'=>1}
+	  assert_equal hash, c.clean_word_hash("here are some good words of test's. I hope you love them!")
+	end
+
+end

data/test/bayes/bayesian_test.rb

@@ -0,0 +1,68 @@
+# coding:utf-8
+
+require File.dirname(__FILE__) + '/../test_helper'
+
+class BayesianTest < Test::Unit::TestCase
+	def setup
+		@classifier = Classifier::Bayes.new :categories => ['Interesting', 'Uninteresting']
+	end
+	
+	def test_good_training
+		assert_nothing_raised { @classifier.train_interesting "love" }
+	end
+
+	def test_bad_training
+		assert_raise(StandardError) { @classifier.train_no_category "words" }
+	end
+	
+	def test_bad_method
+		assert_raise(NoMethodError) { @classifier.forget_everything_you_know "" }
+	end
+	
+	def test_categories
+		assert_equal ['Interesting', 'Uninteresting'].sort, @classifier.categories.sort
+	end
+
+	def test_add_category
+		@classifier.add_category 'Test'
+		assert_equal ['Test', 'Interesting', 'Uninteresting'].sort, @classifier.categories.sort
+	end
+
+	def test_classification
+		@classifier.train_interesting "here are some good words. I hope you love them"
+		@classifier.train_uninteresting "here are some bad words, I hate you"
+		assert_equal 'Uninteresting', @classifier.classify("I hate bad words and you")
+	end
+
+  def test_regression_untrain_nil_fixnum
+    # if a word of the untraining text is not present on the category, a
+    # "TypeError: nil can't be coerced into Fixnum" is raised
+    @classifier.untrain_interesting "nothing"
+  end
+	
+	def test_ru_classification
+	  c = Classifier::Bayes.new :categories => ['Interesting', 'Uninteresting'], :language => "ru"
+	  c.train_interesting "вот несколько хороших слов. Я надеюсь вам они понравились"
+	  c.train_uninteresting "вот несколько плохих слов. Я тебя ненавижу"
+	  assert_equal 'Uninteresting', c.classify("Я ненавижу плохие слова и тебя")
+  end
+  
+  def test_case_insensitive
+	  c = Classifier::Bayes.new :categories => [:good, :bad], :language => "ru"
+	  c.train_good "Хорошо"
+	  c.train_bad "Плохо"
+	  
+	  assert_equal c.classifications("ХОРОШО"), c.classifications("хорошо")
+	  assert_equal c.classifications("плОХО"), c.classifications("плохо")
+  end
+
+  def test_serialize
+    txt = "this can be serialized"
+    @classifier.train_interesting(txt)
+    @classifier.train_uninteresting("really uninteresting")
+
+    b2 = Marshal::load(Marshal::dump(@classifier))
+    assert_equal @classifier.classify(txt), b2.classify(txt)
+  end
+
+end

data/test/lsi/lsi_test.rb

@@ -0,0 +1,167 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class LSITest < Test::Unit::TestCase
+	def setup
+	  # we repeat principle words to help weight them. 
+	  # This test is rather delicate, since this system is mostly noise.
+    @str1 = "This text deals with dogs. Dogs."
+	  @str2 = "This text involves dogs too. Dogs! "
+	  @str3 = "This text revolves around cats. Cats."
+	  @str4 = "This text also involves cats. Cats!"
+	  @str5 = "This text involves birds. Birds."
+	  @str6 = "Is it about dogs or birds?"
+	  @str7 = "Is it about birds or cats?"
+	  @str8 = "I would prefer a bird over thousand cats or dogs because birds are smaller."
+	end
+	
+	def test_basic_indexing
+	 lsi = Classifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+	  assert ! lsi.needs_rebuild?
+	  
+	 # note that the closest match to str1 is str2, even though it is not
+	 # the closest text match.
+	 assert_equal [@str2, @str5, @str3], lsi.find_related(@str1, 3)
+	end
+
+	def test_not_auto_rebuild
+	 lsi = Classifier::LSI.new :auto_rebuild => false
+	 lsi.add_item @str1, "Dog"
+	 lsi.add_item @str2, "Dog"
+	 assert lsi.needs_rebuild?
+	 lsi.build_index
+	 assert ! lsi.needs_rebuild?
+	end
+
+	def test_basic_categorizing_with_too_small_dataset
+	  lsi = Classifier::LSI.new
+	  lsi.add_item @str2, "Dog"
+	  
+	  assert_equal nil, lsi.classify( @str1 )
+	  assert_equal [], lsi.classify_multiple( @str3 )
+	end
+
+	def test_basic_categorizing
+	  lsi = Classifier::LSI.new
+	  lsi.add_item @str2, "Dog"
+	  lsi.add_item @str3, "Cat"
+	  lsi.add_item @str4, "Cat"
+	  lsi.add_item @str5, "Bird"
+	  
+	  assert_equal "Dog", lsi.classify( @str1 )
+	  assert_equal "Cat", lsi.classify( @str3 )
+    assert_equal "Bird", lsi.classify( @str5 )
+    assert_equal "Dog", lsi.classify( @str6 )
+    assert_equal "Bird", lsi.classify( @str7 )
+    assert_equal "Bird", lsi.classify( @str8 )
+	end
+
+  def test_multiple_categorizing
+    lsi = Classifier::LSI.new
+    lsi.add_item @str1, "Dog"
+    lsi.add_item @str2, "Dog"
+    lsi.add_item @str3, "Cat"
+    lsi.add_item @str4, "Cat"
+    lsi.add_item @str5, "Bird"
+
+    assert_equal ["Dog", "Bird"], lsi.classify_multiple( @str6 )
+    assert_equal ["Cat", "Bird"], lsi.classify_multiple( @str7 )
+    assert_equal ["Bird"], lsi.classify_multiple( @str8 )
+  end
+  
+  def test_multiple_categorizing_reverse
+    lsi = Classifier::LSI.new
+    lsi.add_item @str1, "Dog"
+    lsi.add_item @str3, "Cat"
+    lsi.add_item @str4, "Cat"
+    lsi.add_item @str6, "Dog", "Bird", "Flying"
+    lsi.add_item @str7, "Cat", "Bird"
+    lsi.add_item @str8, "Bird", "Dog", "Cat"
+
+    assert_equal ["Dog"], lsi.classify_multiple( @str2 )
+    assert_equal ["Cat", "Bird"], lsi.classify_multiple( @str5 )
+
+    # test with a word unknown alone
+    assert_equal "Bird", lsi.classify( "Bird!" )
+    assert_equal ["Bird", "Dog", "Cat"], lsi.classify_multiple( "Bird!" )
+  end
+  
+	def test_external_classifying
+	  lsi = Classifier::LSI.new
+	  bayes = Classifier::Bayes.new :categories => ['Dog', 'Cat', 'Bird']
+	  lsi.add_item @str1, "Dog" ; bayes.train_dog @str1
+	  lsi.add_item @str2, "Dog" ; bayes.train_dog @str2
+	  lsi.add_item @str3, "Cat" ; bayes.train_cat @str3
+	  lsi.add_item @str4, "Cat" ; bayes.train_cat @str4
+	  lsi.add_item @str5, "Bird" ; bayes.train_bird @str5
+	  
+	  # We're talking about dogs. Even though the text matches the corpus on 
+	  # cats better.  Dogs have more semantic weight than cats. So bayes
+	  # will fail here, but the LSI recognizes content.
+	  tricky_case = "This text revolves around dogs."
+	  assert_equal "Dog", lsi.classify( tricky_case )
+	  assert_not_equal "Dog", bayes.classify( tricky_case )
+	end 
+	
+	def test_recategorize_interface
+	  lsi = Classifier::LSI.new
+	  lsi.add_item @str1, "Dog"
+	  lsi.add_item @str2, "Dog"
+	  lsi.add_item @str3, "Cat"
+	  lsi.add_item @str4, "Cat"
+	  lsi.add_item @str5, "Bird"
+	  
+	  tricky_case = "This text revolves around dogs."
+	  assert_equal "Dog", lsi.classify( tricky_case )
+	  
+	  # Recategorize as needed.
+	  lsi.categories_for(@str1).clear.push "Cow"
+	  lsi.categories_for(@str2).clear.push "Cow"
+
+	  assert !lsi.needs_rebuild?
+	  assert_equal "Cow", lsi.classify( tricky_case )	  
+	end
+	
+	def test_search
+	  lsi = Classifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+	  
+	  # Searching by content and text, note that @str2 comes up first, because
+	  # both "dog" and "involve" are present. But, the next match is @str1 instead
+	  # of @str4, because "dog" carries more weight than involves. 
+	  assert_equal( [@str2, @str1, @str4, @str5, @str3],
+	                lsi.search("dog involves", 100) )
+	                
+	  # Keyword search shows how the space is mapped out in relation to 
+	  # dog when magnitude is remove. Note the relations. We move from dog
+	  # through involve and then finally to other words. 
+	  assert_equal( [@str1, @str2, @str4, @str5, @str3],
+	                lsi.search("dog", 5) )
+	end
+	
+	def test_serialize_safe
+    lsi = Classifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+	  
+	  lsi_md = Marshal.dump lsi
+	  lsi_m = Marshal.load lsi_md
+	  
+	  assert_equal lsi_m.search("cat", 3), lsi.search("cat", 3)
+	  assert_equal lsi_m.find_related(@str1, 3), lsi.find_related(@str1, 3)
+	end
+	
+	def test_keyword_search
+	  lsi = Classifier::LSI.new
+	  lsi.add_item @str1, "Dog"
+	  lsi.add_item @str2, "Dog"
+	  lsi.add_item @str3, "Cat"
+	  lsi.add_item @str4, "Cat"
+	  lsi.add_item @str5, "Bird"
+	
+	  assert_equal ['dog', 'text', 'deal'], lsi.highest_ranked_stems(@str1)
+	end
+	
+	def test_summary
+	   assert_equal "This text involves dogs too [...] This text also involves cats", [@str1, @str2, @str3, @str4, @str5].join.summary(2)
+	end
+	  
+end

data/test/stopwords_test.rb

@@ -0,0 +1,38 @@
+# coding:utf-8
+require File.dirname(__FILE__) + '/test_helper'
+require 'tempfile'
+
+class StopWordsTest < Test::Unit::TestCase
+  def test_en
+    assert_equal 80, Classifier::StopWords.for('en').size
+  end
+
+  def test_ru
+    assert_equal 159, Classifier::StopWords.for('ru').size
+  end
+
+  def test_stopword_es
+    list = Classifier::StopWords.for('es')
+    assert list.include?('más')
+  end
+
+  def test_unknown
+    assert_equal [], Classifier::StopWords.for('_unknown_')
+  end
+
+  def setup
+    @tmp = nil
+  end
+  def teardown
+    Classifier::StopWords.reset
+    File.delete(@tmp) unless @tmp.nil?
+  end
+
+  def test_custom_lang_file
+    lang = 'xxyyzz'
+    @tmp = File.join(File.dirname(__FILE__), lang)
+    File.open(@tmp, 'w') { |f| f.puts "str1\nstr2" }
+    assert_equal ["str1", "str2"], Classifier::StopWords.for(lang,
+      File.dirname(@tmp))
+  end
+end