ankusa 0.0.5 → 0.0.6

data/docs/created.rid

@@ -1 +1 @@
-Fri, 03 Dec 2010 15:29:25 -0500
+Mon, 06 Dec 2010 15:40:49 -0500

data/docs/files/README_rdoc.html

@@ -56,7 +56,7 @@
     </tr>
     <tr class="top-aligned-row">
       <td><strong>Last Update:</strong></td>
-      <td>Thu Dec 02 16:24:11 -0500 2010</td>
+      <td>Mon Dec 06 15:30:41 -0500 2010</td>
     </tr>
     </table>
   </div>
@@ -71,10 +71,16 @@
     <div id="description">
       <h1>ankusa</h1>
 <p>
-<a href="../classes/Ankusa.html">Ankusa</a> is a Naive Bayes classifier in
-Ruby that uses Hadoop&#8216;s HBase for storage. Because it uses HBase as a
+<a href="../classes/Ankusa.html">Ankusa</a> is a text classifier in Ruby
+that uses Hadoop&#8216;s HBase for storage. Because it uses HBase as a
 backend, the training corpus can be many terabytes in size.
 </p>
+<p>
+<a href="../classes/Ankusa.html">Ankusa</a> currently uses a Naive Bayes
+classifier. It ignores common words (a.k.a, stop words) and stems all
+others. Additionally, it uses Laplacian smoothing in the classification
+method.
+</p>
 <h2>Installation</h2>
 <p>
 First, install HBase / Hadoop. Make sure the HBase Thrift interface has
@@ -92,6 +98,8 @@ been started as well. Then:
   storage = Ankusa::HBaseStorage.new 'localhost'
   c = Ankusa::Classifier.new storage
 
+  # Each of these calls will return a bag-of-words
+  # has with stemmed words as keys and counts as values
   c.train :spam, &quot;This is some spammy text&quot;
   c.train :good, &quot;This is not the bad stuff&quot;
 
@@ -102,6 +110,11 @@ been started as well. Then:
   # membership probability as values
   puts c.classifications &quot;This is some spammy text&quot;
 
+  # If you have a large corpus, the probabilities will
+  # likely all be 0.  In that case, you must use log
+  # likelihood values
+  puts c.log_likelihoods &quot;This is some spammy text&quot;
+
   # get a list of all classes
   puts c.classes
 

data/docs/files/lib/ankusa/classifier_rb.html

@@ -56,7 +56,7 @@
     </tr>
     <tr class="top-aligned-row">
       <td><strong>Last Update:</strong></td>
-      <td>Fri Dec 03 07:36:13 -0500 2010</td>
+      <td>Mon Dec 06 15:17:27 -0500 2010</td>
     </tr>
     </table>
   </div>

data/docs/files/lib/ankusa/hbase_storage_rb.html

@@ -56,7 +56,7 @@
     </tr>
     <tr class="top-aligned-row">
       <td><strong>Last Update:</strong></td>
-      <td>Thu Dec 02 13:19:40 -0500 2010</td>
+      <td>Mon Dec 06 15:04:34 -0500 2010</td>
     </tr>
     </table>
   </div>

data/docs/files/lib/ankusa/memory_storage_rb.html

@@ -56,7 +56,7 @@
     </tr>
     <tr class="top-aligned-row">
       <td><strong>Last Update:</strong></td>
-      <td>Thu Dec 02 10:10:26 -0500 2010</td>
+      <td>Mon Dec 06 15:03:20 -0500 2010</td>
     </tr>
     </table>
   </div>

data/docs/fr_method_index.html

@@ -20,47 +20,53 @@
 <div id="index">
   <h1 class="section-bar">Methods</h1>
   <div id="index-entries">
-    <a href="classes/Ankusa/TextHash.html#M000038">add_text (Ankusa::TextHash)</a><br />
-    <a href="classes/Ankusa/TextHash.html#M000039">add_word (Ankusa::TextHash)</a><br />
-    <a href="classes/Ankusa/TextHash.html#M000040">atomize (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000044">add_text (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000045">add_word (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000046">atomize (Ankusa::TextHash)</a><br />
     <a href="classes/Ankusa/Classifier.html#M000007">classifications (Ankusa::Classifier)</a><br />
     <a href="classes/Ankusa/Classifier.html#M000006">classify (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000023">classnames (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000010">classnames (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000034">close (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000021">close (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000020">doc_count_total (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000033">doc_count_total (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000012">drop_tables (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000025">drop_tables (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000036">freq_table (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000016">get_doc_count (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000029">get_doc_count (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000028">get_total_word_count (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000015">get_total_word_count (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000014">get_word_counts (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000027">get_word_counts (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000008">get_word_probs (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000019">incr_doc_count (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000032">incr_doc_count (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000031">incr_total_word_count (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000018">incr_total_word_count (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000030">incr_word_count (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000017">incr_word_count (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000026">init_tables (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000013">init_tables (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/TextHash.html#M000037">new (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000027">classnames (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000013">classnames (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000039">close (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000025">close (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000010">doc_count_totals (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000024">doc_count_totals (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000038">doc_count_totals (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000029">drop_tables (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000015">drop_tables (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000042">freq_table (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000020">get_doc_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000034">get_doc_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000040">get_summary (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000019">get_total_word_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000033">get_total_word_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000032">get_vocabulary_sizes (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000017">get_vocabulary_sizes (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000031">get_word_counts (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000018">get_word_counts (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000009">get_word_probs (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000023">incr_doc_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000037">incr_doc_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000022">incr_total_word_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000036">incr_total_word_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000021">incr_word_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000035">incr_word_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000016">init_tables (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000030">init_tables (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000008">log_likelihoods (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000026">new (Ankusa::HBaseStorage)</a><br />
     <a href="classes/Ankusa/Classifier.html#M000003">new (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000009">new (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000022">new (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000043">new (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000012">new (Ankusa::MemoryStorage)</a><br />
     <a href="classes/String.html#M000001">numeric? (String)</a><br />
-    <a href="classes/Ankusa/MemoryStorage.html#M000011">reset (Ankusa::MemoryStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000024">reset (Ankusa::HBaseStorage)</a><br />
-    <a href="classes/Ankusa/HBaseStorage.html#M000035">summary_table (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000028">reset (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000014">reset (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000041">summary_table (Ankusa::HBaseStorage)</a><br />
     <a href="classes/String.html#M000002">to_ascii (String)</a><br />
     <a href="classes/Ankusa/Classifier.html#M000004">train (Ankusa::Classifier)</a><br />
     <a href="classes/Ankusa/Classifier.html#M000005">untrain (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/TextHash.html#M000041">valid_word? (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000047">valid_word? (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000011">vocab_sizes (Ankusa::Classifier)</a><br />
   </div>
 </div>
 </body>

data/lib/ankusa/classifier.rb

@@ -21,6 +21,9 @@ module Ankusa
       doccount = (text.kind_of? Array) ? text.length : 1
       @storage.incr_doc_count klass, doccount
       @classnames << klass if not @classnames.include? klass
+      # cache is now dirty of these vars
+      @doc_count_totals = nil
+      @vocab_sizes = nil
       th
     end
 
@@ -35,44 +38,70 @@ module Ankusa
       @storage.incr_total_word_count klass, -th.word_count
       doccount = (text.kind_of? Array) ? text.length : 1
       @storage.incr_doc_count klass, -doccount
+      # cache is now dirty of these vars
+      @doc_count_totals = nil
+      @vocab_sizes = nil
       th
     end
 
-    def classify(text)
+    def classify(text, classes=nil)
       # return the most probable class
-      classifications(text).sort_by { |c| -c[1] }.first.first
+      log_likelihoods(text, classes).sort_by { |c| -c[1] }.first.first
     end
     
-    def classifications(text)
+    # Classes is an array of classes to look at
+    def classifications(text, classnames=nil)
+      result = log_likelihoods text, classnames
+      result.keys.each { |k|
+        result[k] = Math.exp result[k] 
+      }
+
+      # normalize to get probs
+      sum = result.values.inject { |x,y| x+y }
+      result.keys.each { |k| result[k] = result[k] / sum }
+      result
+    end
+
+    # Classes is an array of classes to look at
+    def log_likelihoods(text, classnames=nil)
+      classnames ||= @classnames
       result = Hash.new 0
 
       TextHash.new(text).each { |word, count|
-        probs = get_word_probs(word)
-        @classnames.each { |k| result[k] += (Math.log(probs[k]) * count) }
+        probs = get_word_probs(word, classnames)
+        classnames.each { |k| result[k] += (Math.log(probs[k]) * count) }
       }
 
       # add the prior and exponentiate
-      @classnames.each { |k| 
-        result[k] += Math.log(@storage.get_doc_count(k).to_f / @storage.doc_count_total.to_f) 
-        result[k] = Math.exp(result[k])
+      doc_counts = doc_count_totals.select { |k,v| classnames.include? k }.map { |k,v| v }
+      doc_count_total = (doc_counts.inject { |x,y| x+y } + classnames.length).to_f
+      classnames.each { |k| 
+        result[k] += Math.log((@storage.get_doc_count(k) + 1).to_f / doc_count_total) 
       }
       
-      # normalize to get probs
-      sum = result.values.inject { |x,y| x+y }
-      @classnames.each { |k| result[k] = result[k] / sum }
       result
     end
 
     protected
-    def get_word_probs(word)
-      probs = @storage.get_word_counts(word)
-      @classnames.each { |cn| 
+    def get_word_probs(word, classnames)
+      probs = Hash.new 0
+      @storage.get_word_counts(word).each { |k,v| probs[k] = v if classnames.include? k }
+      vs = vocab_sizes
+      classnames.each { |cn| 
         # use a laplacian smoother
-        probs[cn] = (probs[cn] + 1).to_f / (@storage.get_total_word_count(cn) + 1).to_f
+        probs[cn] = (probs[cn] + 1).to_f / (@storage.get_total_word_count(cn) + vs[cn]).to_f
       }
       probs
     end
 
+    def doc_count_totals
+      @doc_count_totals ||= @storage.doc_count_totals
+    end
+
+    def vocab_sizes
+      @vocab_sizes ||= @storage.get_vocabulary_sizes
+    end
+
   end
 
 end

data/lib/ankusa/hbase_storage.rb

@@ -53,12 +53,17 @@ module Ankusa
 
       row.first.columns.each { |colname, cell|
         classname = colname.split(':')[1].intern
-        counts[classname] = cell.to_i64.to_f
+        # in case untrain has been called too many times
+        counts[classname] = [cell.to_i64.to_f, 0].max
       }
 
       counts
     end
 
+    def get_vocabulary_sizes
+      get_summary "totals:vocabsize"
+    end
+
     def get_total_word_count(klass)
       @klass_word_counts.fetch(klass) {
         @klass_word_counts[klass] = summary_table.get(klass, "totals:wordcount").first.to_i64.to_f
@@ -72,7 +77,15 @@ module Ankusa
     end
 
     def incr_word_count(klass, word, count)
-      freq_table.atomic_increment word, "classes:#{klass.to_s}", count
+      size = freq_table.atomic_increment word, "classes:#{klass.to_s}", count
+      # if this is a new word, increase the klass's vocab size.  If the new word
+      # count is 0, then we need to decrement our vocab size
+      if size == count
+        summary_table.atomic_increment klass, "totals:vocabsize"
+      elsif size == 0
+        summary_table.atomic_increment klass, "totals:vocabsize", -1        
+      end
+      size
     end
 
     def incr_total_word_count(klass, count)
@@ -83,12 +96,8 @@ module Ankusa
       @klass_doc_counts[klass] = summary_table.atomic_increment klass, "totals:doccount", count
     end
 
-    def doc_count_total
-      total = 0
-      summary_table.create_scanner("", "totals:doccount") { |row|
-        total += row.columns["totals:doccount"].to_i64
-      }
-      total
+    def doc_count_totals
+      get_summary "totals:doccount"
     end
 
     def close
@@ -96,6 +105,14 @@ module Ankusa
     end
 
     protected
+    def get_summary(name)
+      counts = Hash.new 0
+      summary_table.create_scanner("", name) { |row|
+        counts[row.row.intern] = row.columns[name].to_i64
+      }
+      counts
+    end
+
     def summary_table
       @stable ||= @hbase.get_table @stablename
     end

data/lib/ankusa/memory_storage.rb

@@ -24,6 +24,14 @@ module Ankusa
       @klass_doc_counts = {}
     end
 
+    def get_vocabulary_sizes
+      count = Hash.new 0
+      @freqs.each { |w, ks|
+        ks.keys.each { |k| count[k] += 1 }
+      }
+      count
+    end
+
     def get_word_counts(word)
       @freqs.fetch word, Hash.new(0)
     end
@@ -49,8 +57,8 @@ module Ankusa
       @total_doc_counts[klass] += count
     end
 
-    def doc_count_total
-      @total_doc_counts.values.inject { |x,y| x+y }
+    def doc_count_totals
+      @total_doc_counts
     end
 
     def close

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ankusa
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: false
   segments: 
   - 0
   - 0
-  - 5
-  version: 0.0.5
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - Brian Muller
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-03 00:00:00 -05:00
+date: 2010-12-06 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,7 +50,7 @@ dependencies:
         version: 1.0.0
   type: :runtime
   version_requirements: *id002
-description: Naive Bayes classifier with HBase storage
+description: Text classifier with HBase storage
 email: brian.muller@livingsocial.com
 executables: []
 
@@ -122,6 +122,6 @@ rubyforge_project:
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: Naive Bayes classifier in Ruby that uses Hadoop's HBase for storage
+summary: Text classifier in Ruby that uses Hadoop's HBase for storage
 test_files: []