ankusa 0.0.2 → 0.0.3

data/docs/files/lib/ankusa_rb.html

@@ -56,7 +56,7 @@
     </tr>
     <tr class="top-aligned-row">
       <td><strong>Last Update:</strong></td>
-      <td>Tue Nov 30 12:15:06 -0500 2010</td>
+      <td>Thu Dec 02 07:40:55 -0500 2010</td>
     </tr>
     </table>
   </div>
@@ -73,9 +73,11 @@
       <h3 class="section-bar">Required files</h3>
 
       <div class="name-list">
+      ankusa/extensions&nbsp;&nbsp;
       ankusa/classifier&nbsp;&nbsp;
       ankusa/hasher&nbsp;&nbsp;
-      ankusa/nbclass&nbsp;&nbsp;
+      ankusa/memory_storage&nbsp;&nbsp;
+      ankusa/hbase_storage&nbsp;&nbsp;
       </div>
     </div>
 

data/docs/fr_class_index.html

@@ -22,8 +22,10 @@
   <div id="index-entries">
     <a href="classes/Ankusa.html">Ankusa</a><br />
     <a href="classes/Ankusa/Classifier.html">Ankusa::Classifier</a><br />
-    <a href="classes/Ankusa/NBClass.html">Ankusa::NBClass</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html">Ankusa::HBaseStorage</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html">Ankusa::MemoryStorage</a><br />
     <a href="classes/Ankusa/TextHash.html">Ankusa::TextHash</a><br />
+    <a href="classes/String.html">String</a><br />
   </div>
 </div>
 </body>

data/docs/fr_file_index.html

@@ -23,8 +23,10 @@
     <a href="files/README_rdoc.html">README.rdoc</a><br />
     <a href="files/lib/ankusa_rb.html">lib/ankusa.rb</a><br />
     <a href="files/lib/ankusa/classifier_rb.html">lib/ankusa/classifier.rb</a><br />
+    <a href="files/lib/ankusa/extensions_rb.html">lib/ankusa/extensions.rb</a><br />
     <a href="files/lib/ankusa/hasher_rb.html">lib/ankusa/hasher.rb</a><br />
-    <a href="files/lib/ankusa/nbclass_rb.html">lib/ankusa/nbclass.rb</a><br />
+    <a href="files/lib/ankusa/hbase_storage_rb.html">lib/ankusa/hbase_storage.rb</a><br />
+    <a href="files/lib/ankusa/memory_storage_rb.html">lib/ankusa/memory_storage.rb</a><br />
     <a href="files/lib/ankusa/stopwords_rb.html">lib/ankusa/stopwords.rb</a><br />
   </div>
 </div>

data/docs/fr_method_index.html

@@ -20,23 +20,47 @@
 <div id="index">
   <h1 class="section-bar">Methods</h1>
   <div id="index-entries">
-    <a href="classes/Ankusa/TextHash.html#M000016">add_text (Ankusa::TextHash)</a><br />
-    <a href="classes/Ankusa/TextHash.html#M000017">add_word (Ankusa::TextHash)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000005">classifications (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000004">classify (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000009">doc_count_total (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000007">drop_tables (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000013">freq_table (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000010">get_word_probs (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000011">init_tables (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/NBClass.html#M000014">new (Ankusa::NBClass)</a><br />
-    <a href="classes/Ankusa/TextHash.html#M000015">new (Ankusa::TextHash)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000001">new (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000006">refresh_classnames (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000008">reset (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000012">summary_table (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000002">train (Ankusa::Classifier)</a><br />
-    <a href="classes/Ankusa/Classifier.html#M000003">untrain (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000038">add_text (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000039">add_word (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000040">atomize (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000007">classifications (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000006">classify (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000023">classnames (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000010">classnames (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000034">close (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000021">close (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000020">doc_count_total (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000033">doc_count_total (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000012">drop_tables (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000025">drop_tables (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000036">freq_table (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000016">get_doc_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000029">get_doc_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000028">get_total_word_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000015">get_total_word_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000014">get_word_counts (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000027">get_word_counts (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000008">get_word_probs (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000019">incr_doc_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000032">incr_doc_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000031">incr_total_word_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000018">incr_total_word_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000030">incr_word_count (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000017">incr_word_count (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000026">init_tables (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000013">init_tables (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000037">new (Ankusa::TextHash)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000003">new (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000009">new (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000022">new (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/String.html#M000001">numeric? (String)</a><br />
+    <a href="classes/Ankusa/MemoryStorage.html#M000011">reset (Ankusa::MemoryStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000024">reset (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/Ankusa/HBaseStorage.html#M000035">summary_table (Ankusa::HBaseStorage)</a><br />
+    <a href="classes/String.html#M000002">to_ascii (String)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000004">train (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/Classifier.html#M000005">untrain (Ankusa::Classifier)</a><br />
+    <a href="classes/Ankusa/TextHash.html#M000041">valid_word? (Ankusa::TextHash)</a><br />
   </div>
 </div>
 </body>

data/lib/ankusa.rb

@@ -1,3 +1,5 @@
+require 'ankusa/extensions'
 require 'ankusa/classifier'
 require 'ankusa/hasher'
-require 'ankusa/nbclass'
+require 'ankusa/memory_storage'
+require 'ankusa/hbase_storage'

data/lib/ankusa/classifier.rb

@@ -1,125 +1,76 @@
 module Ankusa
-  SMALL_PROB = 0.0001
 
   class Classifier
     attr_reader :classnames
 
-    def initialize(hbase_client, frequency_tablename="ankusa_word_frequencies", summary_tablename="ankusa_summary")
-      @hbase = hbase_client
-      @ftablename = frequency_tablename
-      @stablename = summary_tablename
-      init_tables
-      @classnames = refresh_classnames
+    def initialize(storage)
+      @storage = storage
+      @storage.init_tables
+      @classnames = @storage.classnames
     end
- 
+
+    # text can be either an array of strings or a string
+    # klass is a symbol
     def train(klass, text)
       th = TextHash.new(text)
       th.each { |word, count|
-        freq_table.atomic_increment word, "classes:#{klass.to_s}", count
+        @storage.incr_word_count klass, word, count
+        yield word, count if block_given?
       }
-      summary_table.atomic_increment klass, "totals:wordcount", th.word_count
-      summary_table.atomic_increment klass, "totals:doccount"
+      @storage.incr_total_word_count klass, th.word_count
+      doccount = (text.kind_of? Array) ? text.length : 1
+      @storage.incr_doc_count klass, doccount
       @classnames << klass if not @classnames.include? klass
     end
 
+    # text can be either an array of strings or a string
+    # klass is a symbol
     def untrain(klass, text)
       th = TextHash.new(text)
       th.each { |word, count|
-        freq_table.atomic_increment word, "classes:#{klass.to_s}", -count
+        @storage.incr_word_count klass, word, -count
+        yield word, count if block_given?
       }
-      summary_table.atomic_increment klass, "totals:wordcount", -th.word_count
-      summary_table.atomic_increment klass, "totals:doccount", -1
+      @storage.incr_total_word_count klass, -th.word_count
+      doccount = (text.kind_of? Array) ? text.length : 1
+      @storage.incr_doc_count klass, -doccount
     end
 
     def classify(text)
       # return the most probable class
-      classifications(text).sort { |o,t| o[1] <=> t[1] }.first.first
+      classifications(text).sort_by { |c| -c[1] }.first.first
     end
     
     def classifications(text)
-      classes = {}
-      result = {}
-      @classnames.each { |k| 
-        classes[k] = NBClass.new k, summary_table, freq_table
-        result[k] = 0 
-      }
+      result = Hash.new 0
 
-      TextHash.new(text).each { |word,count|
-        probs = get_word_probs(word, classes)
-        @classnames.each { |k| result[k] += Math.log(probs[k]) }
+      TextHash.new(text).each { |word, count|
+        probs = get_word_probs(word)
+        @classnames.each { |k| result[k] += (Math.log(probs[k]) * count) }
       }
-     
-      @classnames.each { |k| result[k] += Math.log(classes[k].doc_count / doc_count_total) }
 
-      result.keys.each { |k| result[k] = Math.exp(result[k]) }
-      sum = result.values.inject { |x,y| x+y }
-      result.keys.each { |klass|
-        result[klass] = result[klass] / sum
+      # add the prior and exponentiate
+      @classnames.each { |k| 
+        result[k] += Math.log(@storage.get_doc_count(k).to_f / @storage.doc_count_total.to_f) 
+        result[k] = Math.exp(result[k])
       }
-
+      
+      # normalize to get probs
+      sum = result.values.inject { |x,y| x+y }
+      @classnames.each { |k| result[k] = result[k] / sum }
       result
     end
 
-    # get all classes
-    def refresh_classnames
-      cs = []
-      summary_table.create_scanner("", "totals") { |row|
-        cs << row.row.intern
-      }
-      cs
-    end
-
-    def drop_tables
-      freq_table.delete
-      summary_table.delete      
-      @stable = nil
-      @ftable = nil
-    end
-
-    def reset
-      drop_tables
-      init_tables
-    end
-
-    def doc_count_total
-      total = 0
-      summary_table.create_scanner("", "totals:doccount") { |row|
-        total += row.columns["totals:doccount"].to_i64
-      }
-      total
-    end
-    
     protected
-    def get_word_probs(word, classes)
-      probs = {}
-      @classnames.each { |cn| probs[cn] = Ankusa::SMALL_PROB / classes[cn].word_count }
-      row = freq_table.get_row(word)
-      return probs if row.length == 0
-
-      row.first.columns.each { |colname, cell|
-        classname = colname.split(':')[1].intern
-        probs[classname] = cell.to_i64.to_f / classes[classname].word_count
+    def get_word_probs(word)
+      probs = @storage.get_word_counts(word)
+      @classnames.each { |cn| 
+        # use a laplacian smoother
+        probs[cn] = (probs[cn] + 1).to_f / (@storage.get_total_word_count(cn) + 1).to_f
       }
       probs
     end
 
-    def init_tables
-      if not @hbase.has_table? @ftablename
-        @hbase.create_table @ftablename, "classes", "total"
-      end
-
-      if not @hbase.has_table? @stablename
-        @hbase.create_table @stablename, "totals"
-      end
-    end
-
-    def summary_table
-      @stable ||= @hbase.get_table @stablename
-    end
-
-    def freq_table
-      @ftable ||= @hbase.get_table @ftablename
-    end
   end
 
 end

data/lib/ankusa/extensions.rb

@@ -0,0 +1,13 @@
+require 'iconv'
+
+class String
+  def numeric?
+    true if Float(self) rescue false
+  end
+  
+  def to_ascii
+    # from http://www.jroller.com/obie/tags/unicode
+    converter = Iconv.new('ASCII//IGNORE//TRANSLIT', 'UTF-8') 
+    converter.iconv(self).unpack('U*').select { |cp| cp < 127 }.pack('U*')
+  end
+end

data/lib/ankusa/hasher.rb

@@ -13,21 +13,35 @@ module Ankusa
     end
 
     def add_text(text)
-      # replace dashes with spaces, then get rid of non-word/non-space characters, 
-      # then split by space to get words
-      words = text.tr('-', ' ').gsub(/[^\w\s]/,"").split
-      words.each { |word| add_word word }
+      if text.kind_of? Array
+        text.each { |t| add_text t }
+      else
+        # replace dashes with spaces, then get rid of non-word/non-space characters, 
+        # then split by space to get words
+        words = TextHash.atomize text
+        words.each { |word| add_word(word) if TextHash.valid_word?(word) }
+      end
       self
     end
 
     def add_word(word)
-      word = word.downcase
-      if not Ankusa::STOPWORDS.include? word
-        @word_count += 1
-        key = word.stem.intern
-        store key, fetch(key, 0)+1
-      end
+      @word_count += 1
+      key = word.stem.intern
+      store key, fetch(key, 0)+1
+    end
+
+    def self.atomize(text)
+      text.to_ascii.tr('-', ' ').gsub(/[^\w\s]/," ").split.map { |w| w.downcase }
     end
+
+    # word should be only alphanum chars at this point
+    def self.valid_word?(word)
+      return false if Ankusa::STOPWORDS.include? word
+      return false if word.length < 3
+      return false if word.numeric?
+      true
+    end
+
   end
 
 end

data/lib/ankusa/hbase_storage.rb

@@ -0,0 +1,109 @@
+require 'hbaserb'
+
+module Ankusa
+
+  class HBaseStorage
+    attr_reader :hbase
+
+    def initialize(host='localhost', port=9090, frequency_tablename="ankusa_word_frequencies", summary_tablename="ankusa_summary")
+      @hbase = HBaseRb::Client.new host, port
+      @ftablename = frequency_tablename
+      @stablename = summary_tablename
+      @klass_word_counts = {}
+      @klass_doc_counts = {}
+      init_tables
+    end
+
+    def classnames
+      cs = []
+      summary_table.create_scanner("", "totals") { |row|
+        cs << row.row.intern
+      }
+      cs
+    end
+
+    def reset
+      drop_tables
+      init_tables
+    end
+    
+    def drop_tables
+      freq_table.delete
+      summary_table.delete
+      @stable = nil
+      @ftable = nil
+      @klass_word_counts = {}
+      @klass_doc_counts = {}
+    end
+
+    def init_tables
+      if not @hbase.has_table? @ftablename
+        @hbase.create_table @ftablename, "classes", "total"
+      end
+
+      if not @hbase.has_table? @stablename
+        @hbase.create_table @stablename, "totals"
+      end
+    end
+
+    def get_word_counts(word)
+      counts = Hash.new(0)
+      row = freq_table.get_row(word)
+      return counts if row.length == 0
+
+      row.first.columns.each { |colname, cell|
+        classname = colname.split(':')[1].intern
+        counts[classname] = cell.to_i64.to_f
+      }
+
+      counts
+    end
+
+    def get_total_word_count(klass)
+      @klass_word_counts.fetch(klass) {
+        @klass_word_counts[klass] = summary_table.get(klass, "totals:wordcount").first.to_i64.to_f
+      }
+    end
+    
+    def get_doc_count(klass)
+      @klass_doc_counts.fetch(klass) {
+        @klass_doc_counts[klass] = summary_table.get(klass, "totals:doccount").first.to_i64.to_f
+      }
+    end
+
+    def incr_word_count(klass, word, count)
+      freq_table.atomic_increment word, "classes:#{klass.to_s}", count
+    end
+
+    def incr_total_word_count(klass, count)
+      @klass_word_counts[klass] = summary_table.atomic_increment klass, "totals:wordcount", count
+    end
+
+    def incr_doc_count(klass, count)
+      @klass_doc_counts[klass] = summary_table.atomic_increment klass, "totals:doccount", count
+    end
+
+    def doc_count_total
+      total = 0
+      summary_table.create_scanner("", "totals:doccount") { |row|
+        total += row.columns["totals:doccount"].to_i64
+      }
+      total
+    end
+
+    def close
+      @hbase.close
+    end
+
+    protected
+    def summary_table
+      @stable ||= @hbase.get_table @stablename
+    end
+
+    def freq_table
+      @ftable ||= @hbase.get_table @ftablename
+    end
+
+  end
+
+end