classifier 1.2.0 → 1.3.0

data/doc/classes/Classifier/ContentNode.html

@@ -83,7 +83,7 @@
     <div id="description">
       <p>
 This is an internal data structure class for the <a href="LSI.html">LSI</a>
-node. Save for <a href="ContentNode.html#M000034">raw_vector_with</a>, it
+node. Save for <a href="ContentNode.html#M000049">raw_vector_with</a>, it
 should be fairly straightforward to understand. You should never have to
 use it directly.
 </p>
@@ -97,10 +97,10 @@ use it directly.
       <h3 class="section-bar">Methods</h3>
 
       <div class="name-list">
-      <a href="#M000031">new</a>&nbsp;&nbsp;
-      <a href="#M000034">raw_vector_with</a>&nbsp;&nbsp;
-      <a href="#M000033">search_norm</a>&nbsp;&nbsp;
-      <a href="#M000032">search_vector</a>&nbsp;&nbsp;
+      <a href="#M000046">new</a>&nbsp;&nbsp;
+      <a href="#M000049">raw_vector_with</a>&nbsp;&nbsp;
+      <a href="#M000048">search_norm</a>&nbsp;&nbsp;
+      <a href="#M000047">search_vector</a>&nbsp;&nbsp;
       </div>
     </div>
 
@@ -145,14 +145,9 @@ use it directly.
           <td class="context-item-value">&nbsp;[RW]&nbsp;</td>
           <td class="context-item-desc"></td>
         </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">source</td>
-          <td class="context-item-value">&nbsp;[R]&nbsp;</td>
-          <td class="context-item-desc"></td>
-        </tr>
         <tr class="top-aligned-row context-row">
           <td class="context-item-name">word_hash</td>
-          <td class="context-item-value">&nbsp;[RW]&nbsp;</td>
+          <td class="context-item-value">&nbsp;[R]&nbsp;</td>
           <td class="context-item-desc"></td>
         </tr>
         </table>
@@ -165,13 +160,13 @@ use it directly.
     <div id="methods">
       <h3 class="section-bar">Public Class methods</h3>
 
-      <div id="method-M000031" class="method-detail">
-        <a name="M000031"></a>
+      <div id="method-M000046" class="method-detail">
+        <a name="M000046"></a>
 
         <div class="method-heading">
-          <a href="ContentNode.src/M000031.html" target="Code" class="method-signature"
-            onclick="popupCode('ContentNode.src/M000031.html');return false;">
-          <span class="method-name">new</span><span class="method-args">( source, categories=nil, text_proc=nil )</span>
+          <a href="ContentNode.src/M000046.html" target="Code" class="method-signature"
+            onclick="popupCode('ContentNode.src/M000046.html');return false;">
+          <span class="method-name">new</span><span class="method-args">( word_hash, *categories )</span>
           </a>
         </div>
       
@@ -185,12 +180,12 @@ source.to_s
 
       <h3 class="section-bar">Public Instance methods</h3>
 
-      <div id="method-M000034" class="method-detail">
-        <a name="M000034"></a>
+      <div id="method-M000049" class="method-detail">
+        <a name="M000049"></a>
 
         <div class="method-heading">
-          <a href="ContentNode.src/M000034.html" target="Code" class="method-signature"
-            onclick="popupCode('ContentNode.src/M000034.html');return false;">
+          <a href="ContentNode.src/M000049.html" target="Code" class="method-signature"
+            onclick="popupCode('ContentNode.src/M000049.html');return false;">
           <span class="method-name">raw_vector_with</span><span class="method-args">( word_list )</span>
           </a>
         </div>
@@ -203,12 +198,12 @@ mapping the vector space.
         </div>
       </div>
 
-      <div id="method-M000033" class="method-detail">
-        <a name="M000033"></a>
+      <div id="method-M000048" class="method-detail">
+        <a name="M000048"></a>
 
         <div class="method-heading">
-          <a href="ContentNode.src/M000033.html" target="Code" class="method-signature"
-            onclick="popupCode('ContentNode.src/M000033.html');return false;">
+          <a href="ContentNode.src/M000048.html" target="Code" class="method-signature"
+            onclick="popupCode('ContentNode.src/M000048.html');return false;">
           <span class="method-name">search_norm</span><span class="method-args">()</span>
           </a>
         </div>
@@ -220,12 +215,12 @@ Use this to fetch the appropriate search vector in normalized form.
         </div>
       </div>
 
-      <div id="method-M000032" class="method-detail">
-        <a name="M000032"></a>
+      <div id="method-M000047" class="method-detail">
+        <a name="M000047"></a>
 
         <div class="method-heading">
-          <a href="ContentNode.src/M000032.html" target="Code" class="method-signature"
-            onclick="popupCode('ContentNode.src/M000032.html');return false;">
+          <a href="ContentNode.src/M000047.html" target="Code" class="method-signature"
+            onclick="popupCode('ContentNode.src/M000047.html');return false;">
           <span class="method-name">search_vector</span><span class="method-args">()</span>
           </a>
         </div>

data/doc/classes/Classifier/ContentNode.src/M000046.html

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html>
+<head>
+  <title>new (Classifier::ContentNode)</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
+</head>
+<body class="standalone-code">
+  <pre><span class="ruby-comment cmt"># File lib/classifier/lsi/content_node.rb, line 18</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>( <span class="ruby-identifier">word_hash</span>, <span class="ruby-operator">*</span><span class="ruby-identifier">categories</span> )
+      <span class="ruby-ivar">@categories</span> = <span class="ruby-identifier">categories</span> <span class="ruby-operator">||</span> []
+      <span class="ruby-ivar">@word_hash</span> = <span class="ruby-identifier">word_hash</span>
+    <span class="ruby-keyword kw">end</span></pre>
+</body>
+</html>

data/doc/classes/Classifier/ContentNode.src/{M000032.html → M000047.html}

@@ -10,7 +10,7 @@
   <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
 </head>
 <body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File lib/classifier/lsi/content_node.rb, line 27</span>
+  <pre><span class="ruby-comment cmt"># File lib/classifier/lsi/content_node.rb, line 24</span>
     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">search_vector</span>
       <span class="ruby-ivar">@lsi_vector</span> <span class="ruby-operator">||</span> <span class="ruby-ivar">@raw_vector</span>
     <span class="ruby-keyword kw">end</span></pre>

data/doc/classes/Classifier/ContentNode.src/{M000033.html → M000048.html}

@@ -10,7 +10,7 @@
   <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
 </head>
 <body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File lib/classifier/lsi/content_node.rb, line 32</span>
+  <pre><span class="ruby-comment cmt"># File lib/classifier/lsi/content_node.rb, line 29</span>
     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">search_norm</span>
       <span class="ruby-ivar">@lsi_norm</span> <span class="ruby-operator">||</span> <span class="ruby-ivar">@raw_norm</span>
     <span class="ruby-keyword kw">end</span></pre>

data/doc/classes/Classifier/ContentNode.src/M000049.html

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html>
+<head>
+  <title>raw_vector_with (Classifier::ContentNode)</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
+</head>
+<body class="standalone-code">
+  <pre><span class="ruby-comment cmt"># File lib/classifier/lsi/content_node.rb, line 35</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">raw_vector_with</span>( <span class="ruby-identifier">word_list</span> )
+      <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">$GSL</span>
+         <span class="ruby-identifier">vec</span> = <span class="ruby-constant">Vector</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">word_list</span>.<span class="ruby-identifier">size</span>)
+      <span class="ruby-keyword kw">else</span>
+         <span class="ruby-identifier">vec</span> = <span class="ruby-constant">Array</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">word_list</span>.<span class="ruby-identifier">size</span>, <span class="ruby-value">0</span>)
+      <span class="ruby-keyword kw">end</span>
+
+      <span class="ruby-ivar">@word_hash</span>.<span class="ruby-identifier">each_key</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">word</span><span class="ruby-operator">|</span>
+        <span class="ruby-identifier">vec</span>[<span class="ruby-identifier">word_list</span>[<span class="ruby-identifier">word</span>]] = <span class="ruby-ivar">@word_hash</span>[<span class="ruby-identifier">word</span>] <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">word_list</span>[<span class="ruby-identifier">word</span>]
+      <span class="ruby-keyword kw">end</span>
+     
+      <span class="ruby-comment cmt"># Perform the scaling transform</span>
+      <span class="ruby-identifier">total_words</span> = <span class="ruby-identifier">vec</span>.<span class="ruby-identifier">sum</span>
+      
+      <span class="ruby-comment cmt"># Perform first-order association transform if this vector has more</span>
+      <span class="ruby-comment cmt"># than one word in it. </span>
+      <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">total_words</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">1.0</span> 
+        <span class="ruby-identifier">weighted_total</span> = <span class="ruby-value">0</span><span class="ruby-value">.0</span>
+        <span class="ruby-identifier">vec</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">term</span><span class="ruby-operator">|</span>
+          <span class="ruby-keyword kw">if</span> ( <span class="ruby-identifier">term</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">0</span> )
+            <span class="ruby-identifier">weighted_total</span> <span class="ruby-operator">+=</span> (( <span class="ruby-identifier">term</span> <span class="ruby-operator">/</span> <span class="ruby-identifier">total_words</span> ) <span class="ruby-operator">*</span> <span class="ruby-constant">Math</span>.<span class="ruby-identifier">log</span>( <span class="ruby-identifier">term</span> <span class="ruby-operator">/</span> <span class="ruby-identifier">total_words</span> ))
+          <span class="ruby-keyword kw">end</span>
+        <span class="ruby-keyword kw">end</span> 
+        <span class="ruby-identifier">vec</span> = <span class="ruby-identifier">vec</span>.<span class="ruby-identifier">collect</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">val</span><span class="ruby-operator">|</span> <span class="ruby-constant">Math</span>.<span class="ruby-identifier">log</span>( <span class="ruby-identifier">val</span> <span class="ruby-operator">+</span> <span class="ruby-value">1</span> ) <span class="ruby-operator">/</span> <span class="ruby-operator">-</span><span class="ruby-identifier">weighted_total</span> }
+      <span class="ruby-keyword kw">end</span>
+      
+      <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">$GSL</span>
+         <span class="ruby-ivar">@raw_norm</span>   = <span class="ruby-identifier">vec</span>.<span class="ruby-identifier">normalize</span>
+         <span class="ruby-ivar">@raw_vector</span> = <span class="ruby-identifier">vec</span>
+      <span class="ruby-keyword kw">else</span>
+         <span class="ruby-ivar">@raw_norm</span>   = <span class="ruby-constant">Vector</span>[<span class="ruby-operator">*</span><span class="ruby-identifier">vec</span>].<span class="ruby-identifier">normalize</span>
+         <span class="ruby-ivar">@raw_vector</span> = <span class="ruby-constant">Vector</span>[<span class="ruby-operator">*</span><span class="ruby-identifier">vec</span>]
+      <span class="ruby-keyword kw">end</span>
+    <span class="ruby-keyword kw">end</span></pre>
+</body>
+</html>

data/doc/classes/Classifier/LSI.html

@@ -97,18 +97,22 @@ href="http://en.wikipedia.org/wiki/Latent_Semantic_Indexing">Wikipedia</a>.
       <h3 class="section-bar">Methods</h3>
 
       <div class="name-list">
-      <a href="#M000014">&lt;&lt;</a>&nbsp;&nbsp;
-      <a href="#M000013">add_item</a>&nbsp;&nbsp;
-      <a href="#M000017">build_index</a>&nbsp;&nbsp;
-      <a href="#M000022">classify</a>&nbsp;&nbsp;
-      <a href="#M000021">find_related</a>&nbsp;&nbsp;
-      <a href="#M000016">items</a>&nbsp;&nbsp;
-      <a href="#M000012">needs_rebuild?</a>&nbsp;&nbsp;
-      <a href="#M000011">new</a>&nbsp;&nbsp;
-      <a href="#M000018">proximity_array_for_content</a>&nbsp;&nbsp;
-      <a href="#M000019">proximity_norms_for_content</a>&nbsp;&nbsp;
-      <a href="#M000015">remove_item</a>&nbsp;&nbsp;
-      <a href="#M000020">search</a>&nbsp;&nbsp;
+      <a href="#M000025">&lt;&lt;</a>&nbsp;&nbsp;
+      <a href="#M000024">add_item</a>&nbsp;&nbsp;
+      <a href="#M000030">build_index</a>&nbsp;&nbsp;
+      <a href="#M000029">categories_for</a>&nbsp;&nbsp;
+      <a href="#M000026">categories_for</a>&nbsp;&nbsp;
+      <a href="#M000036">classify</a>&nbsp;&nbsp;
+      <a href="#M000035">find_related</a>&nbsp;&nbsp;
+      <a href="#M000037">highest_ranked_stems</a>&nbsp;&nbsp;
+      <a href="#M000031">highest_relative_content</a>&nbsp;&nbsp;
+      <a href="#M000028">items</a>&nbsp;&nbsp;
+      <a href="#M000023">needs_rebuild?</a>&nbsp;&nbsp;
+      <a href="#M000022">new</a>&nbsp;&nbsp;
+      <a href="#M000032">proximity_array_for_content</a>&nbsp;&nbsp;
+      <a href="#M000033">proximity_norms_for_content</a>&nbsp;&nbsp;
+      <a href="#M000027">remove_item</a>&nbsp;&nbsp;
+      <a href="#M000034">search</a>&nbsp;&nbsp;
       </div>
     </div>
 
@@ -128,6 +132,11 @@ href="http://en.wikipedia.org/wiki/Latent_Semantic_Indexing">Wikipedia</a>.
 
       <div class="name-list">
         <table>
+        <tr class="top-aligned-row context-row">
+          <td class="context-item-name">auto_rebuild</td>
+          <td class="context-item-value">&nbsp;[RW]&nbsp;</td>
+          <td class="context-item-desc"></td>
+        </tr>
         <tr class="top-aligned-row context-row">
           <td class="context-item-name">word_list</td>
           <td class="context-item-value">&nbsp;[R]&nbsp;</td>
@@ -143,12 +152,12 @@ href="http://en.wikipedia.org/wiki/Latent_Semantic_Indexing">Wikipedia</a>.
     <div id="methods">
       <h3 class="section-bar">Public Class methods</h3>
 
-      <div id="method-M000011" class="method-detail">
-        <a name="M000011"></a>
+      <div id="method-M000022" class="method-detail">
+        <a name="M000022"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000011.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000011.html');return false;">
+          <a href="LSI.src/M000022.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000022.html');return false;">
           <span class="method-name">new</span><span class="method-args">(options = {})</span>
           </a>
         </div>
@@ -156,7 +165,7 @@ href="http://en.wikipedia.org/wiki/Latent_Semantic_Indexing">Wikipedia</a>.
         <div class="method-description">
           <p>
 Create a fresh index. If you want to call <a
-href="LSI.html#M000017">build_index</a> manually, use
+href="LSI.html#M000030">build_index</a> manually, use
 </p>
 <pre>
      Classifier::LSI.new :auto_rebuild =&gt; false
@@ -166,31 +175,31 @@ href="LSI.html#M000017">build_index</a> manually, use
 
       <h3 class="section-bar">Public Instance methods</h3>
 
-      <div id="method-M000014" class="method-detail">
-        <a name="M000014"></a>
+      <div id="method-M000025" class="method-detail">
+        <a name="M000025"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000014.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000014.html');return false;">
+          <a href="LSI.src/M000025.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000025.html');return false;">
           <span class="method-name">&lt;&lt;</span><span class="method-args">( item )</span>
           </a>
         </div>
       
         <div class="method-description">
           <p>
-A less flexible shorthand for <a href="LSI.html#M000013">add_item</a> that
+A less flexible shorthand for <a href="LSI.html#M000024">add_item</a> that
 assumes you are passing in a string with no categorries. item will be duck
 typed via to_s .
 </p>
         </div>
       </div>
 
-      <div id="method-M000013" class="method-detail">
-        <a name="M000013"></a>
+      <div id="method-M000024" class="method-detail">
+        <a name="M000024"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000013.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000013.html');return false;">
+          <a href="LSI.src/M000024.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000024.html');return false;">
           <span class="method-name">add_item</span><span class="method-args">( item, *categories, &amp;block )</span>
           </a>
         </div>
@@ -216,12 +225,12 @@ For example:
         </div>
       </div>
 
-      <div id="method-M000017" class="method-detail">
-        <a name="M000017"></a>
+      <div id="method-M000030" class="method-detail">
+        <a name="M000030"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000017.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000017.html');return false;">
+          <a href="LSI.src/M000030.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000030.html');return false;">
           <span class="method-name">build_index</span><span class="method-args">( cutoff=0.75 )</span>
           </a>
         </div>
@@ -229,7 +238,7 @@ For example:
         <div class="method-description">
           <p>
 This function rebuilds the index if <a
-href="LSI.html#M000012">needs_rebuild?</a> returns true. For very large
+href="LSI.html#M000023">needs_rebuild?</a> returns true. For very large
 document spaces, this indexing operation may take some time to complete, so
 it may be wise to place the operation in another thread.
 </p>
@@ -249,12 +258,50 @@ engine.
         </div>
       </div>
 
-      <div id="method-M000022" class="method-detail">
-        <a name="M000022"></a>
+      <div id="method-M000029" class="method-detail">
+        <a name="M000029"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000022.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000022.html');return false;">
+          <a href="LSI.src/M000029.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000029.html');return false;">
+          <span class="method-name">categories_for</span><span class="method-args">(item)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+Returns the categories for a given indexed items. You are free to add and
+remove items from this as you see fit. It does not invalide an index to
+change its categories.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000026" class="method-detail">
+        <a name="M000026"></a>
+
+        <div class="method-heading">
+          <a href="LSI.src/M000026.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000026.html');return false;">
+          <span class="method-name">categories_for</span><span class="method-args">(item)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+Returns the categories for a given indexed items. You are free to add and
+remove items from this as you see fit. It does not invalide an index to
+change its categories.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000036" class="method-detail">
+        <a name="M000036"></a>
+
+        <div class="method-heading">
+          <a href="LSI.src/M000036.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000036.html');return false;">
           <span class="method-name">classify</span><span class="method-args">( doc, cutoff=0.30, &amp;block )</span>
           </a>
         </div>
@@ -263,7 +310,7 @@ engine.
           <p>
 This function uses a voting system to categorize documents, based on the
 categories of other documents. It uses the same logic as the <a
-href="LSI.html#M000021">find_related</a> function to find related
+href="LSI.html#M000035">find_related</a> function to find related
 documents, then returns the most obvious category from this list.
 </p>
 <p>
@@ -274,12 +321,12 @@ the document is in. This may not always make sense.
         </div>
       </div>
 
-      <div id="method-M000021" class="method-detail">
-        <a name="M000021"></a>
+      <div id="method-M000035" class="method-detail">
+        <a name="M000035"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000021.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000021.html');return false;">
+          <a href="LSI.src/M000035.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000035.html');return false;">
           <span class="method-name">find_related</span><span class="method-args">( doc, max_nearest=3, &amp;block )</span>
           </a>
         </div>
@@ -301,12 +348,55 @@ each other in an essay.
         </div>
       </div>
 
-      <div id="method-M000016" class="method-detail">
-        <a name="M000016"></a>
+      <div id="method-M000037" class="method-detail">
+        <a name="M000037"></a>
+
+        <div class="method-heading">
+          <a href="LSI.src/M000037.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000037.html');return false;">
+          <span class="method-name">highest_ranked_stems</span><span class="method-args">( doc, count=3 )</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+Prototype, only works on indexed documents. I have no clue if this is going
+to work, but in theory it&#8217;s supposed to.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000031" class="method-detail">
+        <a name="M000031"></a>
+
+        <div class="method-heading">
+          <a href="LSI.src/M000031.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000031.html');return false;">
+          <span class="method-name">highest_relative_content</span><span class="method-args">( max_chunks=10 )</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+This method returns max_chunks entries, ordered by their average semantic
+rating. Essentially, the average distance of each entry from all other
+entries is calculated, the highest are returned.
+</p>
+<p>
+This can be used to build a summary service, or to provide more information
+about your dataset&#8217;s general content. For example, if you were to use
+categorize on the results of this data, you could gather information on
+what your dataset is generally about.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000028" class="method-detail">
+        <a name="M000028"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000016.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000016.html');return false;">
+          <a href="LSI.src/M000028.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000028.html');return false;">
           <span class="method-name">items</span><span class="method-args">()</span>
           </a>
         </div>
@@ -318,12 +408,12 @@ Returns an array of items that are indexed.
         </div>
       </div>
 
-      <div id="method-M000012" class="method-detail">
-        <a name="M000012"></a>
+      <div id="method-M000023" class="method-detail">
+        <a name="M000023"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000012.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000012.html');return false;">
+          <a href="LSI.src/M000023.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000023.html');return false;">
           <span class="method-name">needs_rebuild?</span><span class="method-args">()</span>
           </a>
         </div>
@@ -337,12 +427,12 @@ classification and cluster detection.
         </div>
       </div>
 
-      <div id="method-M000018" class="method-detail">
-        <a name="M000018"></a>
+      <div id="method-M000032" class="method-detail">
+        <a name="M000032"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000018.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000018.html');return false;">
+          <a href="LSI.src/M000032.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000032.html');return false;">
           <span class="method-name">proximity_array_for_content</span><span class="method-args">( doc, &amp;block )</span>
           </a>
         </div>
@@ -350,7 +440,7 @@ classification and cluster detection.
         <div class="method-description">
           <p>
 This function is the primitive that <a
-href="LSI.html#M000021">find_related</a> and classify build upon. It
+href="LSI.html#M000035">find_related</a> and classify build upon. It
 returns an array of 2-element arrays. The first element of this array is a
 document, and the second is its &quot;score&quot;, defining how
 &quot;close&quot; it is to other indexed items.
@@ -363,25 +453,25 @@ meaningful between indexes.
 <p>
 The parameter doc is the content to compare. If that content is not
 indexed, you can pass an optional block to define how to create the text
-data. See <a href="LSI.html#M000013">add_item</a> for examples of how this
+data. See <a href="LSI.html#M000024">add_item</a> for examples of how this
 works.
 </p>
         </div>
       </div>
 
-      <div id="method-M000019" class="method-detail">
-        <a name="M000019"></a>
+      <div id="method-M000033" class="method-detail">
+        <a name="M000033"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000019.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000019.html');return false;">
+          <a href="LSI.src/M000033.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000033.html');return false;">
           <span class="method-name">proximity_norms_for_content</span><span class="method-args">( doc, &amp;block )</span>
           </a>
         </div>
       
         <div class="method-description">
           <p>
-Similar to <a href="LSI.html#M000018">proximity_array_for_content</a>, this
+Similar to <a href="LSI.html#M000032">proximity_array_for_content</a>, this
 function takes similar arguments and returns a similar array. However, it
 uses the normalized calculated vectors instead of their full versions. This
 is useful when you&#8217;re trying to perform operations on content that is
@@ -391,12 +481,12 @@ primitive.
         </div>
       </div>
 
-      <div id="method-M000015" class="method-detail">
-        <a name="M000015"></a>
+      <div id="method-M000027" class="method-detail">
+        <a name="M000027"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000015.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000015.html');return false;">
+          <a href="LSI.src/M000027.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000027.html');return false;">
           <span class="method-name">remove_item</span><span class="method-args">( item )</span>
           </a>
         </div>
@@ -408,12 +498,12 @@ Removes an item from the database, if it is indexed.
         </div>
       </div>
 
-      <div id="method-M000020" class="method-detail">
-        <a name="M000020"></a>
+      <div id="method-M000034" class="method-detail">
+        <a name="M000034"></a>
 
         <div class="method-heading">
-          <a href="LSI.src/M000020.html" target="Code" class="method-signature"
-            onclick="popupCode('LSI.src/M000020.html');return false;">
+          <a href="LSI.src/M000034.html" target="Code" class="method-signature"
+            onclick="popupCode('LSI.src/M000034.html');return false;">
           <span class="method-name">search</span><span class="method-args">( string, max_nearest=3 )</span>
           </a>
         </div>
@@ -421,7 +511,7 @@ Removes an item from the database, if it is indexed.
         <div class="method-description">
           <p>
 This function allows for text-based search of your index. Unlike other
-functions like <a href="LSI.html#M000021">find_related</a> and classify,
+functions like <a href="LSI.html#M000035">find_related</a> and classify,
 search only takes short strings. It will also ignore factors like repeated
 words. It is best for short, google-like search terms. A search will first
 priortize lexical relationships, then semantic ones.