yury-classifier 1.3.4 → 1.3.5

data/Manifest

@@ -1,3 +1,4 @@
+classifier.gemspec
 lib/classifier/base.rb
 lib/classifier/bayes.rb
 lib/classifier/extensions/vector.rb

data/Rakefile

@@ -2,14 +2,14 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('classifier', '1.3.3') do |p|
+Echoe.new('classifier', '1.3.5') do |p|
   p.description    = "A general classifier module to allow Bayesian and other types of classifications."
   p.url            = "http://github.com/yury/classifier"
   p.author         = "Yury Korolev"
   p.email          = "yury.korolev@gmail.com"
   p.ignore_pattern = ["tmp/*", "script/*"]
   p.development_dependencies = []
-  p.runtime_dependencies = ["activesupport >= 2.2.2", "ruby-stemmer >= 0.5.1"]
+  p.runtime_dependencies = ["activesupport >=2.2.2", "ruby-stemmer >=0.5.1"]
 end
 
 Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }

data/classifier.gemspec

@@ -2,15 +2,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{classifier}
-  s.version = "1.3.4"
+  s.version = "1.3.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Yury Korolev"]
-  s.date = %q{2009-03-22}
+  s.date = %q{2009-04-14}
   s.description = %q{A general classifier module to allow Bayesian and other types of classifications.}
   s.email = %q{yury.korolev@gmail.com}
   s.extra_rdoc_files = ["lib/classifier/base.rb", "lib/classifier/bayes.rb", "lib/classifier/extensions/vector.rb", "lib/classifier/extensions/vector_serialize.rb", "lib/classifier/lsi/content_node.rb", "lib/classifier/lsi/summary.rb", "lib/classifier/lsi/word_list.rb", "lib/classifier/lsi.rb", "lib/classifier.rb", "lib/init.rb", "LICENSE", "README"]
-  s.files = ["lib/classifier/base.rb", "lib/classifier/bayes.rb", "lib/classifier/extensions/vector.rb", "lib/classifier/extensions/vector_serialize.rb", "lib/classifier/lsi/content_node.rb", "lib/classifier/lsi/summary.rb", "lib/classifier/lsi/word_list.rb", "lib/classifier/lsi.rb", "lib/classifier.rb", "lib/init.rb", "LICENSE", "Manifest", "Rakefile", "README", "test/base_test.rb", "test/bayes/bayesian_test.rb", "test/lsi/lsi_test.rb", "test/test_helper.rb", "classifier.gemspec"]
+  s.files = ["classifier.gemspec", "lib/classifier/base.rb", "lib/classifier/bayes.rb", "lib/classifier/extensions/vector.rb", "lib/classifier/extensions/vector_serialize.rb", "lib/classifier/lsi/content_node.rb", "lib/classifier/lsi/summary.rb", "lib/classifier/lsi/word_list.rb", "lib/classifier/lsi.rb", "lib/classifier.rb", "lib/init.rb", "LICENSE", "Manifest", "Rakefile", "README", "test/base_test.rb", "test/bayes/bayesian_test.rb", "test/lsi/lsi_test.rb", "test/test_helper.rb"]
   s.has_rdoc = true
   s.homepage = %q{http://github.com/yury/classifier}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Classifier", "--main", "README"]
@@ -25,14 +25,14 @@ Gem::Specification.new do |s|
     s.specification_version = 2
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<activesupport>, [">= 0", "= 2.2.2"])
-      s.add_runtime_dependency(%q<ruby-stemmer>, [">= 0", "= 0.5.1"])
+      s.add_runtime_dependency(%q<activesupport>, [">= 2.2.2"])
+      s.add_runtime_dependency(%q<ruby-stemmer>, [">= 0.5.1"])
     else
-      s.add_dependency(%q<activesupport>, [">= 0", "= 2.2.2"])
-      s.add_dependency(%q<ruby-stemmer>, [">= 0", "= 0.5.1"])
+      s.add_dependency(%q<activesupport>, [">= 2.2.2"])
+      s.add_dependency(%q<ruby-stemmer>, [">= 0.5.1"])
     end
   else
-    s.add_dependency(%q<activesupport>, [">= 0", "= 2.2.2"])
-    s.add_dependency(%q<ruby-stemmer>, [">= 0", "= 0.5.1"])
+    s.add_dependency(%q<activesupport>, [">= 2.2.2"])
+    s.add_dependency(%q<ruby-stemmer>, [">= 0.5.1"])
   end
 end

data/lib/classifier/lsi/content_node.rb

@@ -54,7 +54,8 @@ module Classifier
           if ( term > 0 )
             weighted_total += (( term / total_words ) * Math.log( term / total_words ))
           end
-        end 
+        end
+        weighted_total = -1.0 if weighted_total.zero? # if no word in list is known
         vec = vec.collect { |val| Math.log( val + 1 ) / -weighted_total }
       end
       

data/lib/classifier/lsi.rb

@@ -180,6 +180,7 @@ module Classifier
       content_node = node_for_content( doc, &block )
       result = 
         @items.keys.collect do |item|
+          next if @items[item].search_vector.blank? # not enough data
           if $GSL
              val = content_node.search_vector * @items[item].search_vector.col
           else
@@ -187,7 +188,7 @@ module Classifier
           end
           [item, val]
         end
-      result.sort_by { |x| x[1] }.reverse
+      result.compact.sort_by { |x| x[1] }.reverse
     end 
     
     # Similar to proximity_array_for_content, this function takes similar
@@ -201,6 +202,7 @@ module Classifier
       content_node = node_for_content( doc, &block )
       result = 
         @items.keys.collect do |item|
+          next if @items[item].search_norm.blank? # not enough data
           if $GSL
             val = content_node.search_norm * @items[item].search_norm.col
           else
@@ -208,7 +210,7 @@ module Classifier
           end
           [item, val]
         end
-      result.sort_by { |x| x[1] }.reverse
+      result.compact.sort_by { |x| x[1] }.reverse
     end 
     
     # This function allows for text-based search of your index. Unlike other functions
@@ -266,7 +268,23 @@ module Classifier
       ranking = votes.keys.sort_by { |x| votes[x] }
       return ranking[-1]
     end
-    
+
+    # Same as previous but returns all results, also more permissive in default cut-off
+    def classify_multiple( doc, cutoff=0.50, &block )
+      icutoff = (@items.size * cutoff).round
+      carry = proximity_array_for_content( doc, &block )
+      carry = carry[0..icutoff-1]
+      votes = {}
+      carry.each do |pair|
+        categories = @items[pair[0]].categories
+        categories.each do |category|
+          votes[category] ||= 0.0
+          votes[category] += pair[1]
+        end
+      end
+      votes.delete_if{|key, value| value<1 }.keys.sort_by { |x| -votes[x] }
+    end
+
     # Prototype, only works on indexed documents.
     # I have no clue if this is going to work, but in theory
     # it's supposed to.

data/test/lsi/lsi_test.rb

@@ -8,6 +8,9 @@ class LSITest < Test::Unit::TestCase
 	  @str3 = "This text revolves around cats. Cats."
 	  @str4 = "This text also involves cats. Cats!"
 	  @str5 = "This text involves birds. Birds."
+	  @str6 = "Is it about dogs or birds?"
+	  @str7 = "Is it about birds or cats?"
+	  @str8 = "I would prefer a bird over thousand cats or dogs because birds are smaller."
 	end
 	
 	def test_basic_indexing
@@ -29,6 +32,14 @@ class LSITest < Test::Unit::TestCase
 	 assert ! lsi.needs_rebuild?
 	end
 
+	def test_basic_categorizing_with_too_small_dataset
+	  lsi = Classifier::LSI.new
+	  lsi.add_item @str2, "Dog"
+	  
+	  assert_equal nil, lsi.classify( @str1 )
+	  assert_equal [], lsi.classify_multiple( @str3 )
+	end
+
 	def test_basic_categorizing
 	  lsi = Classifier::LSI.new
 	  lsi.add_item @str2, "Dog"
@@ -38,9 +49,42 @@ class LSITest < Test::Unit::TestCase
 	  
 	  assert_equal "Dog", lsi.classify( @str1 )
 	  assert_equal "Cat", lsi.classify( @str3 )
-    assert_equal "Bird", lsi.classify( @str5 )  
+    assert_equal "Bird", lsi.classify( @str5 )
+    assert_equal "Dog", lsi.classify( @str6 )
+    assert_equal "Bird", lsi.classify( @str7 )
+    assert_equal "Bird", lsi.classify( @str8 )
 	end
-	
+
+  def test_multiple_categorizing
+    lsi = Classifier::LSI.new
+    lsi.add_item @str1, "Dog"
+    lsi.add_item @str2, "Dog"
+    lsi.add_item @str3, "Cat"
+    lsi.add_item @str4, "Cat"
+    lsi.add_item @str5, "Bird"
+
+    assert_equal ["Dog", "Bird"], lsi.classify_multiple( @str6 )
+    assert_equal ["Cat", "Bird"], lsi.classify_multiple( @str7 )
+    assert_equal ["Bird"], lsi.classify_multiple( @str8 )
+  end
+  
+  def test_multiple_categorizing_reverse
+    lsi = Classifier::LSI.new
+    lsi.add_item @str1, "Dog"
+    lsi.add_item @str3, "Cat"
+    lsi.add_item @str4, "Cat"
+    lsi.add_item @str6, "Dog", "Bird", "Flying"
+    lsi.add_item @str7, "Cat", "Bird"
+    lsi.add_item @str8, "Bird", "Dog", "Cat"
+
+    assert_equal ["Dog"], lsi.classify_multiple( @str2 )
+    assert_equal ["Cat", "Bird"], lsi.classify_multiple( @str5 )
+
+    # test with a word unknown alone
+    assert_equal "Bird", lsi.classify( "Bird!" )
+    assert_equal ["Bird", "Dog", "Cat"], lsi.classify_multiple( "Bird!" )
+  end
+  
 	def test_external_classifying
 	  lsi = Classifier::LSI.new
 	  bayes = Classifier::Bayes.new :categories => ['Dog', 'Cat', 'Bird']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: yury-classifier
 version: !ruby/object:Gem::Version 
-  version: 1.3.4
+  version: 1.3.5
 platform: ruby
 authors: 
 - Yury Korolev
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-22 00:00:00 -07:00
+date: 2009-04-14 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -19,9 +19,6 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    - - "="
       - !ruby/object:Gem::Version 
         version: 2.2.2
     version: 
@@ -32,9 +29,6 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    - - "="
       - !ruby/object:Gem::Version 
         version: 0.5.1
     version: 
@@ -58,6 +52,7 @@ extra_rdoc_files:
 - LICENSE
 - README
 files: 
+- classifier.gemspec
 - lib/classifier/base.rb
 - lib/classifier/bayes.rb
 - lib/classifier/extensions/vector.rb
@@ -76,7 +71,6 @@ files:
 - test/bayes/bayesian_test.rb
 - test/lsi/lsi_test.rb
 - test/test_helper.rb
-- classifier.gemspec
 has_rdoc: true
 homepage: http://github.com/yury/classifier
 post_install_message: