ankusa 0.0.6 → 0.0.7

data/lib/ankusa.rb

@@ -1,5 +1,5 @@
 require 'ankusa/extensions'
 require 'ankusa/classifier'
+require 'ankusa/naive_bayes'
+require 'ankusa/kl_divergence'
 require 'ankusa/hasher'
-require 'ankusa/memory_storage'
-require 'ankusa/hbase_storage'

data/lib/ankusa/cassandra_storage.rb

@@ -0,0 +1,194 @@
+require 'cassandra/0.7'
+
+#
+# At the moment you'll have to do:
+#
+# create keyspace ankusa with replication_factor = 1
+#
+# from the cassandra-cli. This should be fixed with new release candidate for
+# cassandra
+#
+module Ankusa
+
+  class CassandraStorage
+    attr_reader :cassandra
+
+    #
+    # Necessary to set max classes since current implementation of ruby
+    # cassandra client doesn't support table scans. Using crufty get_range
+    # method at the moment.
+    #
+    def initialize(host='127.0.0.1', port=9160, keyspace = 'ankusa', max_classes = 100)
+      @cassandra  = Cassandra.new('system', "#{host}:#{port}")
+      @klass_word_counts = {}
+      @klass_doc_counts  = {}
+      @keyspace    = keyspace
+      @max_classes = max_classes
+      init_tables
+    end
+
+    #
+    # Fetch the names of the distinct classes for classification:
+    # eg. :spam, :good, etc
+    #
+    def classnames
+      @cassandra.get_range(:totals, {:start => '', :finish => '', :count => @max_classes}).inject([]) do |cs, key_slice|
+        cs << key_slice.key.to_sym
+      end
+    end
+
+    def reset
+      drop_tables
+      init_tables
+    end
+
+    #
+    # Drop ankusa keyspace, reset internal caches
+    #
+    # FIXME: truncate doesn't work with cassandra-beta2
+    #
+    def drop_tables
+      @cassandra.truncate!('classes')
+      @cassandra.truncate!('totals')
+      @cassandra.drop_keyspace(@keyspace)
+      @klass_word_counts = {}
+      @klass_doc_counts  = {}
+    end
+
+
+    #
+    # Create required keyspace and column families
+    #
+    def init_tables
+      # Do nothing if keyspace already exists
+      if @cassandra.keyspaces.include?(@keyspace)
+        @cassandra.keyspace = @keyspace
+      else
+        freq_table    = Cassandra::ColumnFamily.new({:keyspace => @keyspace, :name => "classes"}) # word  => {classname => count}
+        summary_table = Cassandra::ColumnFamily.new({:keyspace => @keyspace, :name => "totals"})  # class => {wordcount => count}
+        ks_def = Cassandra::Keyspace.new({
+            :name               => @keyspace,
+            :strategy_class     => 'org.apache.cassandra.locator.SimpleStrategy',
+            :replication_factor => 1,
+            :cf_defs            => [freq_table, summary_table] 
+          })
+        @cassandra.add_keyspace ks_def
+        @cassandra.keyspace = @keyspace
+      end
+    end
+
+    #
+    # Fetch hash of word counts as a single row from cassandra.
+    # Here column_name is the class and column value is the count
+    #
+    def get_word_counts(word)
+      # fetch all (class,count) pairs for a given word
+      row = @cassandra.get(:classes, word.to_s)
+      return row.to_hash if row.empty?
+      row.inject({}){|counts, col| counts[col.first.to_sym] = [col.last.to_f,0].max; counts}
+    end
+
+    #
+    # Does a table 'scan' of summary table pulling out the 'vocabsize' column
+    # from each row. Generates a hash of (class, vocab_size) key value pairs
+    #
+    def get_vocabulary_sizes
+      get_summary "vocabsize"
+    end
+
+    #
+    # Fetch total word count for a given class and cache it
+    #
+    def get_total_word_count(klass)
+      @klass_word_counts[klass] = @cassandra.get(:totals, klass.to_s, "wordcount").values.last.to_f
+    end
+
+    #
+    # Fetch total documents for a given class and cache it
+    #
+    def get_doc_count(klass)
+      @klass_doc_counts[klass] = @cassandra.get(:totals, klass.to_s, "doc_count").values.last.to_f
+    end
+
+    #
+    # Increment the count for a given (word,class) pair. Evidently, cassandra
+    # does not support atomic increment/decrement. Psh. HBase uses ZooKeeper to
+    # implement atomic operations, ain't it special?
+    #
+    def incr_word_count(klass, word, count)
+      # Only wants strings
+      klass = klass.to_s
+      word  = word.to_s
+
+      prior_count = @cassandra.get(:classes, word, klass).values.last.to_i
+      new_count   = prior_count + count
+      @cassandra.insert(:classes, word, {klass => new_count.to_s})
+
+      if (prior_count == 0 && count > 0)
+        #
+        # we've never seen this word before and we're not trying to unlearn it
+        #
+        vocab_size = @cassandra.get(:totals, klass, "vocabsize").values.last.to_i
+        vocab_size += 1
+        @cassandra.insert(:totals, klass, {"vocabsize" => vocab_size.to_s})
+      elsif new_count == 0
+        #
+        # we've seen this word before but we're trying to unlearn it
+        #
+        vocab_size = @cassandra.get(:totals, klass, "vocabsize").values.last.to_i
+        vocab_size -= 1
+        @cassandra.insert(:totals, klass, {"vocabsize" => vocab_size.to_s})
+      end
+      new_count
+    end
+
+    #
+    # Increment total word count for a given class by 'count'
+    #
+    def incr_total_word_count(klass, count)
+      klass = klass.to_s
+      wordcount = @cassandra.get(:totals, klass, "wordcount").values.last.to_i
+      wordcount += count
+      @cassandra.insert(:totals, klass, {"wordcount" => wordcount.to_s})
+      @klass_word_counts[klass.to_sym] = wordcount
+    end
+
+    #
+    # Increment total document count for a given class by 'count'
+    #
+    def incr_doc_count(klass, count)
+      klass = klass.to_s
+      doc_count = @cassandra.get(:totals, klass, "doc_count").values.last.to_i
+      doc_count += count
+      @cassandra.insert(:totals, klass, {"doc_count" => doc_count.to_s})
+      @klass_doc_counts[klass.to_sym] = doc_count
+    end
+
+    def doc_count_totals
+      get_summary "doc_count"
+    end
+
+    #
+    # Doesn't do anything
+    #
+    def close
+    end
+
+    protected
+
+    #
+    # Fetch 100 rows from summary table, yes, increase if necessary
+    #
+    def get_summary(name)
+      counts = {}
+      @cassandra.get_range(:totals, {:start => '', :finish => '', :count => @max_classes}).each do |key_slice|
+        # keyslice is a clunky thrift object, map into a ruby hash
+        row = key_slice.columns.inject({}){|hsh, c| hsh[c.column.name] = c.column.value; hsh}
+        counts[key_slice.key.to_sym] = row[name].to_f
+      end
+      counts
+    end
+
+  end
+
+end

data/lib/ankusa/classifier.rb

@@ -1,6 +1,6 @@
 module Ankusa
 
-  class Classifier
+  module Classifier
     attr_reader :classnames
 
     def initialize(storage)
@@ -44,44 +44,6 @@ module Ankusa
       th
     end
 
-    def classify(text, classes=nil)
-      # return the most probable class
-      log_likelihoods(text, classes).sort_by { |c| -c[1] }.first.first
-    end
-    
-    # Classes is an array of classes to look at
-    def classifications(text, classnames=nil)
-      result = log_likelihoods text, classnames
-      result.keys.each { |k|
-        result[k] = Math.exp result[k] 
-      }
-
-      # normalize to get probs
-      sum = result.values.inject { |x,y| x+y }
-      result.keys.each { |k| result[k] = result[k] / sum }
-      result
-    end
-
-    # Classes is an array of classes to look at
-    def log_likelihoods(text, classnames=nil)
-      classnames ||= @classnames
-      result = Hash.new 0
-
-      TextHash.new(text).each { |word, count|
-        probs = get_word_probs(word, classnames)
-        classnames.each { |k| result[k] += (Math.log(probs[k]) * count) }
-      }
-
-      # add the prior and exponentiate
-      doc_counts = doc_count_totals.select { |k,v| classnames.include? k }.map { |k,v| v }
-      doc_count_total = (doc_counts.inject { |x,y| x+y } + classnames.length).to_f
-      classnames.each { |k| 
-        result[k] += Math.log((@storage.get_doc_count(k) + 1).to_f / doc_count_total) 
-      }
-      
-      result
-    end
-
     protected
     def get_word_probs(word, classnames)
       probs = Hash.new 0

data/lib/ankusa/kl_divergence.rb

@@ -0,0 +1,31 @@
+module Ankusa
+
+  class KLDivergenceClassifier
+    include Classifier
+
+    def classify(text, classes=nil)
+      # return the class with the least distance from the word
+      # distribution of the given text
+      distances(text, classes).sort_by { |c| c[1] }.first.first
+    end
+    
+
+    # Classes is an array of classes to look at
+    def distances(text, classnames=nil)
+      classnames ||= @classnames
+      distances = Hash.new 0
+
+      th = TextHash.new(text)
+      th.each { |word, count|
+        thprob = count.to_f / th.length.to_f
+        probs = get_word_probs(word, classnames)
+        classnames.each { |k| 
+          distances[k] += (thprob * Math.log(thprob / probs[k]) * count) 
+        }
+      }
+
+      distances
+    end
+  end
+
+end

data/lib/ankusa/naive_bayes.rb

@@ -0,0 +1,46 @@
+module Ankusa
+
+  class NaiveBayesClassifier
+    include Classifier
+
+    def classify(text, classes=nil)
+      # return the most probable class
+      log_likelihoods(text, classes).sort_by { |c| -c[1] }.first.first
+    end
+    
+    # Classes is an array of classes to look at
+    def classifications(text, classnames=nil)
+      result = log_likelihoods text, classnames
+      result.keys.each { |k|
+        result[k] = Math.exp result[k] 
+      }
+
+      # normalize to get probs
+      sum = result.values.inject { |x,y| x+y }
+      result.keys.each { |k| result[k] = result[k] / sum }
+      result
+    end
+
+    # Classes is an array of classes to look at
+    def log_likelihoods(text, classnames=nil)
+      classnames ||= @classnames
+      result = Hash.new 0
+
+      TextHash.new(text).each { |word, count|
+        probs = get_word_probs(word, classnames)
+        classnames.each { |k| result[k] += (Math.log(probs[k]) * count) }
+      }
+
+      # add the prior and exponentiate
+      doc_counts = doc_count_totals.select { |k,v| classnames.include? k }.map { |k,v| v }
+      doc_count_total = (doc_counts.inject { |x,y| x+y } + classnames.length).to_f
+      classnames.each { |k| 
+        result[k] += Math.log((@storage.get_doc_count(k) + 1).to_f / doc_count_total) 
+      }
+      
+      result
+    end
+
+  end
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ankusa
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: false
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 7
+  version: 0.0.7
 platform: ruby
 authors: 
 - Brian Muller
@@ -15,29 +15,13 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-06 00:00:00 -05:00
+date: 2010-12-12 00:00:00 -05:00
 default_executable: 
 dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hbaserb
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 25
-        segments: 
-        - 0
-        - 0
-        - 3
-        version: 0.0.3
-  type: :runtime
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: fast-stemmer
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -49,8 +33,8 @@ dependencies:
         - 0
         version: 1.0.0
   type: :runtime
-  version_requirements: *id002
-description: Text classifier with HBase storage
+  version_requirements: *id001
+description: Text classifier with HBase or Cassandra storage
 email: brian.muller@livingsocial.com
 executables: []
 
@@ -59,28 +43,37 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- lib/ankusa/cassandra_storage.rb
 - lib/ankusa/classifier.rb
 - lib/ankusa/extensions.rb
 - lib/ankusa/hasher.rb
 - lib/ankusa/hbase_storage.rb
+- lib/ankusa/kl_divergence.rb
 - lib/ankusa/memory_storage.rb
+- lib/ankusa/naive_bayes.rb
 - lib/ankusa/stopwords.rb
 - lib/ankusa.rb
 - LICENSE
 - Rakefile
 - README.rdoc
+- docs/classes/Ankusa/CassandraStorage.html
 - docs/classes/Ankusa/Classifier.html
 - docs/classes/Ankusa/HBaseStorage.html
+- docs/classes/Ankusa/KLDivergenceClassifier.html
 - docs/classes/Ankusa/MemoryStorage.html
+- docs/classes/Ankusa/NaiveBayesClassifier.html
 - docs/classes/Ankusa/TextHash.html
 - docs/classes/Ankusa.html
 - docs/classes/String.html
 - docs/created.rid
+- docs/files/lib/ankusa/cassandra_storage_rb.html
 - docs/files/lib/ankusa/classifier_rb.html
 - docs/files/lib/ankusa/extensions_rb.html
 - docs/files/lib/ankusa/hasher_rb.html
 - docs/files/lib/ankusa/hbase_storage_rb.html
+- docs/files/lib/ankusa/kl_divergence_rb.html
 - docs/files/lib/ankusa/memory_storage_rb.html
+- docs/files/lib/ankusa/naive_bayes_rb.html
 - docs/files/lib/ankusa/stopwords_rb.html
 - docs/files/lib/ankusa_rb.html
 - docs/files/README_rdoc.html
@@ -116,12 +109,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       segments: 
       - 0
       version: "0"
-requirements: []
-
+requirements: 
+- Either hbaserb >= 0.0.3 or cassandra >= 0.7
 rubyforge_project: 
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: Text classifier in Ruby that uses Hadoop's HBase for storage
+summary: Text classifier in Ruby that uses Hadoop's HBase or Cassandra for storage
 test_files: []