reclassifier 0.0.2

data/test/lsi_test.rb

@@ -0,0 +1,123 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class LSITest < Test::Unit::TestCase
+	def setup
+	  # we repeat principle words to help weight them.
+	  # This test is rather delicate, since this system is mostly noise.
+     @str1 = "This text deals with dogs. Dogs."
+	  @str2 = "This text involves dogs too. Dogs! "
+	  @str3 = "This text revolves around cats. Cats."
+	  @str4 = "This text also involves cats. Cats!"
+	  @str5 = "This text involves birds. Birds."
+	end
+
+	def test_basic_indexing
+	 lsi = Reclassifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+	  assert ! lsi.needs_rebuild?
+
+	 # note that the closest match to str1 is str2, even though it is not
+	 # the closest text match.
+	 assert_equal [@str2, @str5, @str3], lsi.find_related(@str1, 3)
+	end
+
+	def test_not_auto_rebuild
+	 lsi = Reclassifier::LSI.new :auto_rebuild => false
+	 lsi.add_item @str1, "Dog"
+	 lsi.add_item @str2, "Dog"
+	 assert lsi.needs_rebuild?
+	 lsi.build_index
+	 assert ! lsi.needs_rebuild?
+	end
+
+	def test_basic_categorizing
+	  lsi = Reclassifier::LSI.new
+	  lsi.add_item @str2, "Dog"
+	  lsi.add_item @str3, "Cat"
+	  lsi.add_item @str4, "Cat"
+	  lsi.add_item @str5, "Bird"
+
+	  assert_equal "Dog", lsi.classify( @str1 )
+	  assert_equal "Cat", lsi.classify( @str3 )
+     assert_equal "Bird", lsi.classify( @str5 )
+	end
+
+	def test_external_classifying
+	  lsi = Reclassifier::LSI.new
+	  bayes = Reclassifier::Bayes.new 'Dog', 'Cat', 'Bird'
+	  lsi.add_item @str1, "Dog" ; bayes.train_dog @str1
+	  lsi.add_item @str2, "Dog" ; bayes.train_dog @str2
+	  lsi.add_item @str3, "Cat" ; bayes.train_cat @str3
+	  lsi.add_item @str4, "Cat" ; bayes.train_cat @str4
+	  lsi.add_item @str5, "Bird" ; bayes.train_bird @str5
+
+	  # We're talking about dogs. Even though the text matches the corpus on
+	  # cats better.  Dogs have more semantic weight than cats. So bayes
+	  # will fail here, but the LSI recognizes content.
+	  tricky_case = "This text revolves around dogs."
+	  assert_equal "Dog", lsi.classify( tricky_case )
+	  assert_not_equal "Dog", bayes.classify( tricky_case )
+	end
+
+	def test_recategorize_interface
+	  lsi = Reclassifier::LSI.new
+	  lsi.add_item @str1, "Dog"
+	  lsi.add_item @str2, "Dog"
+	  lsi.add_item @str3, "Cat"
+	  lsi.add_item @str4, "Cat"
+	  lsi.add_item @str5, "Bird"
+
+	  tricky_case = "This text revolves around dogs."
+	  assert_equal "Dog", lsi.classify( tricky_case )
+
+	  # Recategorize as needed.
+	  lsi.categories_for(@str1).clear.push "Cow"
+	  lsi.categories_for(@str2).clear.push "Cow"
+
+	  assert !lsi.needs_rebuild?
+	  assert_equal "Cow", lsi.classify( tricky_case )
+	end
+
+	def test_search
+	  lsi = Reclassifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+
+	  # Searching by content and text, note that @str2 comes up first, because
+	  # both "dog" and "involve" are present. But, the next match is @str1 instead
+	  # of @str4, because "dog" carries more weight than involves.
+	  assert_equal( [@str2, @str1, @str4, @str5, @str3],
+	                lsi.search("dog involves", 100) )
+
+	  # Keyword search shows how the space is mapped out in relation to
+	  # dog when magnitude is remove. Note the relations. We move from dog
+	  # through involve and then finally to other words.
+	  assert_equal( [@str1, @str2, @str4, @str5, @str3],
+	                lsi.search("dog", 5) )
+	end
+
+	def test_serialize_safe
+    lsi = Reclassifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+
+	  lsi_md = Marshal.dump lsi
+	  lsi_m = Marshal.load lsi_md
+
+	  assert_equal lsi_m.search("cat", 3), lsi.search("cat", 3)
+	  assert_equal lsi_m.find_related(@str1, 3), lsi.find_related(@str1, 3)
+	end
+
+	def test_keyword_search
+	  lsi = Reclassifier::LSI.new
+	  lsi.add_item @str1, "Dog"
+	  lsi.add_item @str2, "Dog"
+	  lsi.add_item @str3, "Cat"
+	  lsi.add_item @str4, "Cat"
+	  lsi.add_item @str5, "Bird"
+
+	  assert_equal [:dog, :text, :deal], lsi.highest_ranked_stems(@str1)
+	end
+
+	def test_summary
+	   assert_equal "This text involves dogs too [...] This text also involves cats", [@str1, @str2, @str3, @str4, @str5].join.summary(2)
+	end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'test/unit'
+require 'reclassifier'

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: reclassifier
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Ryan Oblak
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fast-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gsl
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Bayesian and Latent Semantic Indexing classification of text.
+email:
+- rroblak@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/gsl/vector.rb
+- lib/reclassifier.rb
+- lib/reclassifier/bayes.rb
+- lib/reclassifier/content_node.rb
+- lib/reclassifier/core_ext/array.rb
+- lib/reclassifier/core_ext/matrix.rb
+- lib/reclassifier/core_ext/object.rb
+- lib/reclassifier/core_ext/string.rb
+- lib/reclassifier/core_ext/vector.rb
+- lib/reclassifier/lsi.rb
+- lib/reclassifier/version.rb
+- lib/reclassifier/word_list.rb
+- reclassifier.gemspec
+- test/bayes_test.rb
+- test/core_ext/array_test.rb
+- test/core_ext/string_test.rb
+- test/lsi_test.rb
+- test/test_helper.rb
+homepage: https://github.com/saveup/reclassifier
+licenses:
+- LGPL
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Bayesian and Latent Semantic Indexing classification of text.
+test_files:
+- test/bayes_test.rb
+- test/core_ext/array_test.rb
+- test/core_ext/string_test.rb
+- test/lsi_test.rb
+- test/test_helper.rb