classifier 1.1.1 → 1.2.0

data/lib/classifier/string_extensions.rb

@@ -2,10 +2,15 @@
 # Copyright:: Copyright (c) 2005 Lucas Carlson
 # License::   LGPL
 
-require 'classifier/string_extensions/porter_stemmer'
-require 'classifier/string_extensions/word_hash'
+begin
+	require_gem 'stemmer'
+rescue LoadError
+	puts "Please install stemmer from http://rubyforge.org/projects/stemmer or 'gem install stemmer'"
+	exit(-1)
+end
 
-class String
-	include Classifier::Stemmable
-	include Classifier::WordHash
+require 'classifier/extensions/word_hash'
+
+class Object
+	def prepare_category_name; to_s.gsub("_"," ").capitalize.intern end
 end

data/test/bayes/bayesian_test.rb

@@ -17,12 +17,12 @@ class BayesianTest < Test::Unit::TestCase
 	end
 	
 	def test_categories
-		assert_equal ['Interesting', 'Uninteresting'], @classifier.categories
+		assert_equal ['Interesting', 'Uninteresting'].sort, @classifier.categories.sort
 	end
 
 	def test_add_category
 		@classifier.add_category 'Test'
-		assert_equal ['Interesting', 'Test', 'Uninteresting'], @classifier.categories
+		assert_equal ['Test', 'Interesting', 'Uninteresting'].sort, @classifier.categories.sort
 	end
 
 	def test_classification

data/test/lsi/lsi_test.rb

@@ -0,0 +1,88 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class LSITest < Test::Unit::TestCase
+	def setup
+	  # we repeat principle words to help weight them. 
+	  # This test is rather delicate, since this system is mostly noise.
+    @str1 = "This text deals with dogs. Dogs."
+	  @str2 = "This text involves dogs too. Dogs! "
+	  @str3 = "This text revolves around cats. Cats."
+	  @str4 = "This text also involves cats. Cats!"
+	  @str5 = "This text involves birds. Birds."
+	end
+	
+	def test_basic_indexing
+	 lsi = Classifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+	  assert ! lsi.needs_rebuild?
+	  
+	 # note that the closest match to str1 is str2, even though it is not
+	 # the closest text match.
+	 assert_equal [@str2, @str5, @str3], lsi.find_related(@str1, 3)
+	end
+
+	def test_not_auto_rebuild
+	 lsi = Classifier::LSI.new :auto_rebuild => false
+	 lsi.add_item @str1, "Dog"
+	 assert lsi.needs_rebuild?
+	 lsi.build_index
+	 assert ! lsi.needs_rebuild?
+	end
+
+	def test_basic_categorizing
+	  lsi = Classifier::LSI.new
+	  lsi.add_item @str2, "Dog"
+	  lsi.add_item @str3, "Cat"
+	  lsi.add_item @str4, "Cat"
+	  lsi.add_item @str5, "Bird"
+	  
+	  assert_equal "Dog", lsi.classify( @str1 )
+	  assert_equal "Cat", lsi.classify( @str3 )
+     assert_equal "Bird", lsi.classify( @str5 )  
+	end
+	
+	def test_external_classifying
+	  lsi = Classifier::LSI.new
+	  bayes = Classifier::Bayes.new 'Dog', 'Cat', 'Bird'
+	  lsi.add_item @str1, "Dog" ; bayes.train_dog @str1
+	  lsi.add_item @str2, "Dog" ; bayes.train_dog @str2
+	  lsi.add_item @str3, "Cat" ; bayes.train_cat @str3
+	  lsi.add_item @str4, "Cat" ; bayes.train_cat @str4
+	  lsi.add_item @str5, "Bird" ; bayes.train_bird @str5
+	  
+	  # We're talking about dogs. Even though the text matches the corpus on 
+	  # cats better.  Dogs have more semantic weight than cats. So bayes
+	  # will fail here, but the LSI recognizes content.
+	  tricky_case = "This text revolves around dogs."
+	  assert_equal "Dog", lsi.classify( tricky_case )
+	  assert_not_equal "Dog", bayes.classify( tricky_case )
+	end 
+	
+	def test_search
+	  lsi = Classifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+	  
+	  # Searching by content and text, note that @str2 comes up first, because
+	  # both "dog" and "involve" are present. But, the next match is @str1 instead
+	  # of @str4, because "dog" carries more weight than involves. 
+	  assert_equal( [@str2, @str1, @str4, @str5, @str3],
+	                lsi.search("dog involves", 100) )
+	                
+	  # Keyword search shows how the space is mapped out in relation to 
+	  # dog when magnitude is remove. Note the relations. We move from dog
+	  # through involve and then finally to other words. 
+	  assert_equal( [@str1, @str2, @str4, @str5, @str3],
+	                lsi.search("dog", 5) )
+	end
+	
+	def test_serialize_safe
+    lsi = Classifier::LSI.new
+	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
+	  
+	  lsi_md = Marshal.dump lsi
+	  lsi_m = Marshal.load lsi_md
+	  
+	  assert_equal lsi_m.search("cat", 3), lsi.search("cat", 3)
+	  assert_equal lsi_m.find_related(@str1, 3), lsi.find_related(@str1, 3)
+	end
+	
+end

data/test/string_extensions/word_hash_test.rb

@@ -1,12 +1,14 @@
 require File.dirname(__FILE__) + '/../test_helper'
 class StringExtensionsTest < Test::Unit::TestCase
 	def test_word_hash
-		hash = {:good=>1, :"!"=>1, :hope=>1, :"'."=>1, :love=>1, :word=>1, :them=>1, :test=>1}
+		hash = {:good=>1, :"!"=>1, :hope=>1, :"'"=>1, :"."=>1, :love=>1, :word=>1, :them=>1, :test=>1}
 		assert_equal hash, "here are some good words of test's. I hope you love them!".word_hash
 	end
-	
-	def test_without_punctuation
-	  sample = "Hello  thats welcome    to   no    punctuation"
-	  assert_equal sample, "Hello, that's welcome () to ! no *& punctuation".without_punctuation
+
+   	
+	def test_clean_word_hash
+	   hash = {:good=>1, :word=>1, :hope=>1, :love=>1, :them=>1, :test=>1}
+	   assert_equal hash, "here are some good words of test's. I hope you love them!".clean_word_hash
 	end
+
 end

metadata

@@ -3,13 +3,13 @@ rubygems_version: 0.8.6
 specification_version: 1
 name: classifier
 version: !ruby/object:Gem::Version 
-  version: 1.1.1
-date: 2005-04-11
+  version: 1.2.0
+date: 2005-04-24
 summary: A general classifier module to allow Bayesian and other types of classifications.
 require_paths: 
   - lib
 email: lucas@rufy.com
-homepage: http://rubyforge.org/projects/classifier/
+homepage: http://classifier.rufy.com/
 rubyforge_project: 
 description: A general classifier module to allow Bayesian and other types of classifications.
 autorequire: classifier
@@ -30,16 +30,23 @@ files:
   - lib/classifier
   - lib/classifier.rb
   - lib/classifier/bayes.rb
-  - lib/classifier/string_extensions
+  - lib/classifier/extensions
+  - lib/classifier/lsi
+  - lib/classifier/lsi.rb
   - lib/classifier/string_extensions.rb
-  - lib/classifier/string_extensions/porter_stemmer.rb
-  - lib/classifier/string_extensions/word_hash.rb
+  - lib/classifier/extensions/vector_serialize.rb
+  - lib/classifier/extensions/word_hash.rb
+  - lib/classifier/extensions/word_list.rb
+  - lib/classifier/lsi/content_node.rb
   - bin/bayes.rb
   - test/bayes
+  - test/lsi
   - test/string_extensions
   - test/test_helper.rb
   - test/bayes/bayesian_test.rb
+  - test/lsi/lsi_test.rb
   - test/string_extensions/word_hash_test.rb
+  - LICENSE
   - Rakefile
   - README
   - doc/classes
@@ -52,35 +59,83 @@ files:
   - doc/rdoc-style.css
   - doc/classes/Classifier
   - doc/classes/Classifier.html
+  - doc/classes/GSL
+  - doc/classes/GSL.html
+  - doc/classes/Object.html
+  - doc/classes/Object.src
   - doc/classes/String.html
+  - doc/classes/String.src
   - doc/classes/Classifier/Bayes.html
   - doc/classes/Classifier/Bayes.src
-  - doc/classes/Classifier/Stemmable.html
-  - doc/classes/Classifier/Stemmable.src
-  - doc/classes/Classifier/WordHash.html
-  - doc/classes/Classifier/WordHash.src
-  - doc/classes/Classifier/Bayes.src/M000005.html
-  - doc/classes/Classifier/Bayes.src/M000006.html
-  - doc/classes/Classifier/Bayes.src/M000007.html
-  - doc/classes/Classifier/Bayes.src/M000008.html
-  - doc/classes/Classifier/Bayes.src/M000009.html
-  - doc/classes/Classifier/Bayes.src/M000010.html
-  - doc/classes/Classifier/Stemmable.src/M000003.html
-  - doc/classes/Classifier/WordHash.src/M000001.html
-  - doc/classes/Classifier/WordHash.src/M000002.html
+  - doc/classes/Classifier/ContentNode.html
+  - doc/classes/Classifier/ContentNode.src
+  - doc/classes/Classifier/LSI.html
+  - doc/classes/Classifier/LSI.src
+  - doc/classes/Classifier/WordList.html
+  - doc/classes/Classifier/WordList.src
+  - doc/classes/Classifier/Bayes.src/M000023.html
+  - doc/classes/Classifier/Bayes.src/M000024.html
+  - doc/classes/Classifier/Bayes.src/M000025.html
+  - doc/classes/Classifier/Bayes.src/M000026.html
+  - doc/classes/Classifier/Bayes.src/M000027.html
+  - doc/classes/Classifier/Bayes.src/M000028.html
+  - doc/classes/Classifier/Bayes.src/M000029.html
+  - doc/classes/Classifier/ContentNode.src/M000031.html
+  - doc/classes/Classifier/ContentNode.src/M000032.html
+  - doc/classes/Classifier/ContentNode.src/M000033.html
+  - doc/classes/Classifier/ContentNode.src/M000034.html
+  - doc/classes/Classifier/LSI.src/M000011.html
+  - doc/classes/Classifier/LSI.src/M000012.html
+  - doc/classes/Classifier/LSI.src/M000013.html
+  - doc/classes/Classifier/LSI.src/M000014.html
+  - doc/classes/Classifier/LSI.src/M000015.html
+  - doc/classes/Classifier/LSI.src/M000016.html
+  - doc/classes/Classifier/LSI.src/M000017.html
+  - doc/classes/Classifier/LSI.src/M000018.html
+  - doc/classes/Classifier/LSI.src/M000019.html
+  - doc/classes/Classifier/LSI.src/M000020.html
+  - doc/classes/Classifier/LSI.src/M000021.html
+  - doc/classes/Classifier/LSI.src/M000022.html
+  - doc/classes/Classifier/WordList.src/M000007.html
+  - doc/classes/Classifier/WordList.src/M000008.html
+  - doc/classes/Classifier/WordList.src/M000009.html
+  - doc/classes/Classifier/WordList.src/M000010.html
+  - doc/classes/GSL/Vector.html
+  - doc/classes/GSL/Vector.src
+  - doc/classes/GSL/Vector.src/M000005.html
+  - doc/classes/GSL/Vector.src/M000006.html
+  - doc/classes/Object.src/M000001.html
+  - doc/classes/String.src/M000002.html
+  - doc/classes/String.src/M000003.html
+  - doc/classes/String.src/M000004.html
   - doc/files/lib
   - doc/files/README.html
   - doc/files/lib/classifier
   - doc/files/lib/classifier_rb.html
   - doc/files/lib/classifier/bayes_rb.html
-  - doc/files/lib/classifier/string_extensions
+  - doc/files/lib/classifier/extensions
+  - doc/files/lib/classifier/lsi
+  - doc/files/lib/classifier/lsi_rb.html
   - doc/files/lib/classifier/string_extensions_rb.html
-  - doc/files/lib/classifier/string_extensions/porter_stemmer_rb.html
-  - doc/files/lib/classifier/string_extensions/word_hash_rb.html
+  - doc/files/lib/classifier/extensions/vector_serialize_rb.html
+  - doc/files/lib/classifier/extensions/word_hash_rb.html
+  - doc/files/lib/classifier/extensions/word_list_rb.html
+  - doc/files/lib/classifier/lsi/content_node_rb.html
 test_files: []
 rdoc_options: []
 extra_rdoc_files: []
 executables: []
 extensions: []
-requirements: []
-dependencies: []
+requirements: 
+  - A porter-stemmer module to split word stems.
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: stemmer
+    version_requirement: 
+    version_requirements: !ruby/object:Gem::Version::Requirement 
+      requirements: 
+        - 
+          - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+      version: 

data/doc/classes/Classifier/Stemmable.html

@@ -1,243 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html 
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>Module: Classifier::Stemmable</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <meta http-equiv="Content-Script-Type" content="text/javascript" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-  <script type="text/javascript">
-  // <![CDATA[
-
-  function popupCode( url ) {
-    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
-  }
-
-  function toggleCode( id ) {
-    if ( document.getElementById )
-      elem = document.getElementById( id );
-    else if ( document.all )
-      elem = eval( "document.all." + id );
-    else
-      return false;
-
-    elemStyle = elem.style;
-    
-    if ( elemStyle.display != "block" ) {
-      elemStyle.display = "block"
-    } else {
-      elemStyle.display = "none"
-    }
-
-    return true;
-  }
-  
-  // Make codeblocks hidden by default
-  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
-  
-  // ]]>
-  </script>
-
-</head>
-<body>
-
-
-
-    <div id="classHeader">
-        <table class="header-table">
-        <tr class="top-aligned-row">
-          <td><strong>Module</strong></td>
-          <td class="class-name-in-header">Classifier::Stemmable</td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>In:</strong></td>
-            <td>
-                <a href="../../files/lib/classifier/string_extensions/porter_stemmer_rb.html">
-                lib/classifier/string_extensions/porter_stemmer.rb
-                </a>
-        <br />
-            </td>
-        </tr>
-
-        </table>
-    </div>
-  <!-- banner header -->
-
-  <div id="bodyContent">
-
-
-
-  <div id="contextContent">
-
-    <div id="description">
-      <p>
-Porter stemmer in Ruby.
-</p>
-<p>
-This is the Porter stemming algorithm, ported to Ruby from the version
-coded up in Perl. It&#8217;s easy to follow against the rules in the
-original paper in:
-</p>
-<pre>
-  Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
-  no. 3, pp 130-137,
-</pre>
-<p>
-See also <a
-href="http://www.tartarus.org/~martin/PorterStemmer">www.tartarus.org/~martin/PorterStemmer</a>
-</p>
-<p>
-Send comments to raypereda@hotmail.com
-</p>
-
-    </div>
-
-
-   </div>
-
-    <div id="method-list">
-      <h3 class="section-bar">Methods</h3>
-
-      <div class="name-list">
-      <a href="#M000004">stem</a>&nbsp;&nbsp;
-      <a href="#M000003">stem_porter</a>&nbsp;&nbsp;
-      </div>
-    </div>
-
-  </div>
-
-
-    <!-- if includes -->
-
-    <div id="section">
-
-
-    <div id="constants-list">
-      <h3 class="section-bar">Constants</h3>
-
-      <div class="name-list">
-        <table summary="Constants">
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">STEP_2_LIST</td>
-          <td>=</td>
-          <td class="context-item-value">{
     'ational'=&gt;'ate', 'tional'=&gt;'tion', 'enci'=&gt;'ence', 'anci'=&gt;'ance',
     'izer'=&gt;'ize', 'bli'=&gt;'ble',
     'alli'=&gt;'al', 'entli'=&gt;'ent', 'eli'=&gt;'e', 'ousli'=&gt;'ous',
     'ization'=&gt;'ize', 'ation'=&gt;'ate',
     'ator'=&gt;'ate', 'alism'=&gt;'al', 'iveness'=&gt;'ive', 'fulness'=&gt;'ful',
     'ousness'=&gt;'ous', 'aliti'=&gt;'al',
     'iviti'=&gt;'ive', 'biliti'=&gt;'ble', 'logi'=&gt;'log'</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">STEP_3_LIST</td>
-          <td>=</td>
-          <td class="context-item-value">{
     'icate'=&gt;'ic', 'ative'=&gt;'', 'alize'=&gt;'al', 'iciti'=&gt;'ic',
     'ical'=&gt;'ic', 'ful'=&gt;'', 'ness'=&gt;''</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">SUFFIX_1_REGEXP</td>
-          <td>=</td>
-          <td class="context-item-value">/(
                     ational  |
                     tional   |
                     enci     |
                     anci     |
                     izer     |
                     bli      |
                     alli     |
                     entli    |
                     eli      |
                     ousli    |
                     ization  |
                     ation    |
                     ator     |
                     alism    |
                     iveness  |
                     fulness  |
                     ousness  |
                     aliti    |
                     iviti    |
                     biliti   |
                     logi)$/x</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">SUFFIX_2_REGEXP</td>
-          <td>=</td>
-          <td class="context-item-value">/(
                       al       |
                       ance     |
                       ence     |
                       er       |
                       ic       | 
                       able     |
                       ible     |
                       ant      |
                       ement    |
                       ment     |
                       ent      |
                       ou       |
                       ism      |
                       ate      |
                       iti      |
                       ous      |
                       ive      |
                       ize)$/x</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">C</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;[^aeiou]&quot;</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">V</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;[aeiouy]&quot;</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">CC</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;#{C}(?&gt;[^aeiouy]*)&quot;</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">VV</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;#{V}(?&gt;[aeiou]*)&quot;</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">MGR0</td>
-          <td>=</td>
-          <td class="context-item-value">/^(#{CC})?#{VV}#{CC}/o</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">MEQ1</td>
-          <td>=</td>
-          <td class="context-item-value">/^(#{CC})?#{VV}#{CC}(#{VV})?$/o</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">MGR1</td>
-          <td>=</td>
-          <td class="context-item-value">/^(#{CC})?#{VV}#{CC}#{VV}#{CC}/o</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">VOWEL_IN_STEM</td>
-          <td>=</td>
-          <td class="context-item-value">/^(#{CC})?#{V}/o</td>
-        </tr>
-        </table>
-      </div>
-    </div>
-
-
-
-      
-
-
-    <!-- if method_list -->
-    <div id="methods">
-      <h3 class="section-bar">Public Instance methods</h3>
-
-      <div id="method-M000004" class="method-detail">
-        <a name="M000004"></a>
-
-        <div class="method-heading">
-          <span class="method-name">stem</span><span class="method-args">()</span>
-        </div>
-      
-        <div class="method-description">
-          <p>
-Alias for <a href="Stemmable.html#M000003">stem_porter</a>
-</p>
-        </div>
-      </div>
-
-      <div id="method-M000003" class="method-detail">
-        <a name="M000003"></a>
-
-        <div class="method-heading">
-          <a href="Stemmable.src/M000003.html" target="Code" class="method-signature"
-            onclick="popupCode('Stemmable.src/M000003.html');return false;">
-          <span class="method-name">stem_porter</span><span class="method-args">()</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-          <p>
-Stems the word contained in the current object. E.g.,
-</p>
-<pre>
-    &quot;actually&quot;.stem_porter
-    =&gt; &quot;actual&quot;
-</pre>
-        </div>
-      </div>
-
-
-    </div>
-
-
-  </div>
-
-
-<div id="validator-badges">
-  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-</div>
-
-</body>
-</html>