reclassifier 0.4.5 → 0.4.6

data/lib/reclassifier/bayes.rb

@@ -2,8 +2,11 @@
 # Bayesian classifier for arbitrary text.
 #
 # Implementation is translated from
-# <em>Introduction to Information Retrieval</em> by Christopher D. Manning, Prabhakar Raghavan and Hinrich Schütze,
-# Cambridge University Press. 2008, ISBN 0521865719.
+# <em>Introduction to Information Retrieval</em> by Christopher D. Manning,
+# Prabhakar Raghavan and Hinrich Schütze, # Cambridge University Press. 2008,
+# ISBN 0521865719.
+#
+# Derived quantities are cached to improve performance of repeated #classify calls.
 #
 class Reclassifier::Bayes
   include Reclassifier::WordHash
@@ -33,7 +36,7 @@ class Reclassifier::Bayes
   def train(classification, text)
     ensure_classification_exists(classification)
 
-    @docs_in_classification_count[classification] += 1
+    update_doc_count(classification, 1)
 
     smart_word_hash(text).each do |word, count|
       @classifications[classification][word] ||= 0
@@ -53,7 +56,7 @@ class Reclassifier::Bayes
   def untrain(classification, text)
     ensure_classification_exists(classification)
 
-    @docs_in_classification_count[classification] -= 1
+    update_doc_count(classification, -1)
 
     smart_word_hash(text).each do |word, count|
       @classifications[classification][word] -= count if @classifications[classification].include?(word)
@@ -68,17 +71,21 @@ class Reclassifier::Bayes
   def calculate_scores(text)
     scores = {}
 
+    @cache[:total_docs_classified_log] ||= Math.log(@docs_in_classification_count.values.reduce(:+))
+    @cache[:words_classified] ||= @classifications.values.reduce(Set.new) {|set, word_counts| set.merge(word_counts.keys)}
+
     @classifications.each do |classification, classification_word_counts|
       # prior
       scores[classification] = Math.log(@docs_in_classification_count[classification])
-      scores[classification] -= Math.log(@docs_in_classification_count.values.reduce(:+))
+      scores[classification] -= @cache[:total_docs_classified_log]
 
       # likelihood
+      classification_word_count = classification_word_counts.values.reduce(:+).to_i
       smart_word_hash(text).each do |word, count|
-        if @classifications.values.reduce(Set.new) {|set, word_counts| set.merge(word_counts.keys)}.include?(word)
+        if @cache[:words_classified].include?(word)
           scores[classification] += count * Math.log((classification_word_counts[word] || 0) + 1)
 
-          scores[classification] -= count * Math.log(classification_word_counts.values.reduce(:+).to_i + @classifications.values.reduce(Set.new) {|set, word_counts| set.merge(word_counts.keys)}.count)
+          scores[classification] -= count * Math.log(classification_word_count + @cache[:words_classified].count)
         end
       end
     end
@@ -135,7 +142,52 @@ class Reclassifier::Bayes
     return_value
   end
 
+  # Invalidates the cache.
+  #
+  #   classifier = Reclassifier::Bayes.new([:one, :other])
+  #
+  #   classifier.train(:one, 'bbb')
+  #   classifier.train(:other, 'aaa')
+  #
+  #   classifier.classify('aaa')
+  #
+  #   classifier.cache_set?
+  #   =>  true
+  #
+  #   classifier.invalidate_cache
+  #
+  #   classifier.cache_set?
+  #   =>  false
+  #
+  def invalidate_cache
+    @cache = {}
+  end
+
+  # Returns true if the cache has been set (i.e. #classify has been run).
+  # Returns false otherwise.
+  #   classifier = Reclassifier::Bayes.new([:one, :other])
+  #
+  #   classifier.cache_set?
+  #   =>  false
+  #
+  #   classifier.train(:one, 'bbb')
+  #   classifier.train(:other, 'aaa')
+  #
+  #   classifier.classify('aaa')
+  #
+  #   classifier.cache_set?
+  #   =>  true
+  #
+  def cache_set?
+    @cache.present?
+  end
+
   private
+    def update_doc_count(classification, value)
+      @docs_in_classification_count[classification] += value
+
+      invalidate_cache
+    end
 
     def ensure_classification_exists(classification)
       raise Reclassifier::UnknownClassificationError unless @classifications.include?(classification)

data/lib/reclassifier/version.rb

@@ -1,3 +1,3 @@
 module Reclassifier
-  VERSION = "0.4.5"
+  VERSION = "0.4.6"
 end

data/lib/reclassifier.rb

@@ -2,6 +2,7 @@
 require 'fast-stemmer'
 require 'gsl'
 require 'matrix'
+require 'active_support/core_ext/object/blank'
 
 # files
 require 'reclassifier/version'

data/reclassifier.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'fast-stemmer'
   spec.add_dependency 'gsl'
+  spec.add_dependency 'activesupport'
 end

data/spec/bayes_spec.rb

@@ -1,134 +1,184 @@
 require 'spec_helper'
 
 describe Reclassifier::Bayes do
+  subject(:classifier) { Reclassifier::Bayes.new }
+
+  shared_examples 'cache invalidator' do |method|
+    it 'should invalidate the cache' do
+      classifier = Reclassifier::Bayes.new([:in_china, :not_in_china])
+
+      classifier.should_receive(:invalidate_cache)
+
+      classifier.send(method, :in_china, 'Chinese Beijing Chinese')
+    end
+  end
+
 	describe "classifications" do
     it "should return the classifications" do
-      subject = described_class.new([:interesting, :uninteresting])
+      classifier = Reclassifier::Bayes.new([:interesting, :uninteresting])
 
-      subject.classifications.sort.should eq([:interesting, :uninteresting])
+      classifier.classifications.sort.should eq([:interesting, :uninteresting])
     end
 	end
 
   describe "train" do
     it "should raise an UnknownClassificationError if the specified classification hasn't been added" do
-      expect {subject.train(:blargle, '')}.to raise_error(Reclassifier::UnknownClassificationError)
+      expect { classifier.train(:blargle, '')}.to raise_error(Reclassifier::UnknownClassificationError)
     end
 
     it "should train the classifier to the (classification, document) pair" do
-      subject = described_class.new([:in_china, :not_in_china])
+      classifier = Reclassifier::Bayes.new([:in_china, :not_in_china])
 
-      subject.train(:in_china, 'Chinese Beijing Chinese')
-      subject.train(:in_china, 'Chinese Chinese Shanghai')
-      subject.train(:in_china, 'Chinese Macao')
-      subject.train(:not_in_china, 'Tokyo Japan Chinese')
+      classifier.train(:in_china, 'Chinese Beijing Chinese')
+      classifier.train(:in_china, 'Chinese Chinese Shanghai')
+      classifier.train(:in_china, 'Chinese Macao')
+      classifier.train(:not_in_china, 'Tokyo Japan Chinese')
 
-      subject.classify('Chinese Chinese Chinese Tokyo Japan').should eq(:in_china)
+      classifier.classify('Chinese Chinese Chinese Tokyo Japan').should eq(:in_china)
     end
+
+    it_should_behave_like 'cache invalidator', :train
   end
 
-  describe "untrain" do
+  describe 'untrain' do
     it "should raise an UnknownClassificationError if the specified classification hasn't been added" do
-      expect {subject.untrain(:blargle, '')}.to raise_error(Reclassifier::UnknownClassificationError)
+      expect {classifier.untrain(:blargle, '')}.to raise_error(Reclassifier::UnknownClassificationError)
     end
 
-    it "should untrain the classifier against the (classification, document) pair" do
-      subject = described_class.new([:in_china, :not_in_china])
+    it 'should untrain the classifier against the (classification, document) pair' do
+      classifier = Reclassifier::Bayes.new([:in_china, :not_in_china])
 
-      subject.train(:in_china, 'Chinese Chinese')
-      subject.train(:not_in_china, 'Chinese Macao')
+      classifier.train(:in_china, 'Chinese Chinese')
+      classifier.train(:not_in_china, 'Chinese Macao')
 
-      subject.classify('Chinese').should eq(:in_china)
+      classifier.classify('Chinese').should eq(:in_china)
 
-      subject.untrain(:in_china, 'Chinese Chinese')
+      classifier.untrain(:in_china, 'Chinese Chinese')
 
-      subject.classify('Chinese').should eq(:not_in_china)
+      classifier.classify('Chinese').should eq(:not_in_china)
     end
+
+    it 'should not result in negative word counts'
+
+    it_should_behave_like 'cache invalidator', :untrain
   end
 
   describe "calculate_scores" do
     it "should return a score hash with the correct scores" do
-      subject = described_class.new([:in_china, :not_in_china])
+      classifier = Reclassifier::Bayes.new([:in_china, :not_in_china])
 
-      subject.train(:in_china, 'Chinese Beijing Chinese')
-      subject.train(:in_china, 'Chinese Chinese Shanghai')
-      subject.train(:in_china, 'Chinese Macao')
-      subject.train(:not_in_china, 'Tokyo Japan Chinese')
+      classifier.train(:in_china, 'Chinese Beijing Chinese')
+      classifier.train(:in_china, 'Chinese Chinese Shanghai')
+      classifier.train(:in_china, 'Chinese Macao')
+      classifier.train(:not_in_china, 'Tokyo Japan Chinese')
 
-      scores = subject.calculate_scores('Chinese Chinese Chinese Tokyo Japan')
+      scores = classifier.calculate_scores('Chinese Chinese Chinese Tokyo Japan')
 
       scores[:in_china].should eq(-8.107690312843907)
       scores[:not_in_china].should eq(-8.906681345001262)
     end
 
     it "should handle the case when no documents are classified for a particular classification" do
-      subject = described_class.new([:in_china, :not_in_china])
+      classifier = Reclassifier::Bayes.new([:in_china, :not_in_china])
 
-      subject.train(:in_china, 'Chinese Beijing Chinese')
+      classifier.train(:in_china, 'Chinese Beijing Chinese')
 
-      subject.calculate_scores('Chinese Beijing')
+      classifier.calculate_scores('Chinese Beijing')
     end
   end
 
   describe "add_classification" do
     it "should add the classification to the set of classifications" do
-      subject.classifications.should be_empty
+      classifier.classifications.should be_empty
 
-      subject.add_classification(:niner)
+      classifier.add_classification(:niner)
 
-      subject.classifications.should eq([:niner])
+      classifier.classifications.should eq([:niner])
     end
 
     it "should return the classification" do
-      subject.add_classification(:niner).should eq(:niner)
+      classifier.add_classification(:niner).should eq(:niner)
     end
   end
 
   describe "remove_classification" do
     it "should remove the classification from the set of classifications" do
-      subject.add_classification(:niner)
+      classifier.add_classification(:niner)
 
-      subject.remove_classification(:niner)
+      classifier.remove_classification(:niner)
 
-      subject.classifications.should be_empty
+      classifier.classifications.should be_empty
     end
 
     it "should return the classification" do
-      subject.add_classification(:niner)
+      classifier.add_classification(:niner)
 
-      subject.remove_classification(:niner).should eq(:niner)
+      classifier.remove_classification(:niner).should eq(:niner)
     end
 
     it "should return nil if the classification didn't exist" do
-      subject.remove_classification(:niner).should be(nil)
+      classifier.remove_classification(:niner).should be(nil)
+    end
+  end
+
+  describe 'cache_present?' do
+    it 'should return true if the cache has been set' do
+      classifier = Reclassifier::Bayes.new([:one, :other])
+
+      classifier.train(:one, 'bbb')
+      classifier.train(:other, 'aaa')
+
+      classifier.classify('')
+
+      classifier.cache_set?.should be(true)
+    end
+
+    it 'should return false if the cache has not been set' do
+      classifier.cache_set?.should be(false)
+    end
+
+    it 'should return false if the cache has been invalidated' do
+      classifier = Reclassifier::Bayes.new([:one, :other])
+
+      classifier.train(:one, 'bbb')
+      classifier.train(:other, 'aaa')
+
+      classifier.classify('')
+
+      classifier.cache_set?.should be(true)
+
+      classifier.invalidate_cache
+
+      classifier.cache_set?.should be(false)
     end
   end
 
   context ':clean option' do
     it 'should cause punctuation to be omitted if it is set to true' do
-      subject = described_class.new([:one, :other], {:clean => true})
+      classifier = Reclassifier::Bayes.new([:one, :other], {:clean => true})
 
-      subject.train(:one, '! ! ! ! bbb')
-      subject.train(:other, 'aaa')
+      classifier.train(:one, '! ! ! ! bbb')
+      classifier.train(:other, 'aaa')
 
-      subject.classify('! aaa !').should eq(:other)
+      classifier.classify('! aaa !').should eq(:other)
     end
 
     it 'should default to true' do
-      subject = described_class.new([:one, :other])
+      classifier = Reclassifier::Bayes.new([:one, :other])
 
-      subject.train(:one, '! ! ! ! bbb')
-      subject.train(:other, 'aaa')
+      classifier.train(:one, '! ! ! ! bbb')
+      classifier.train(:other, 'aaa')
 
-      subject.classify('! aaa !').should eq(:other)
+      classifier.classify('! aaa !').should eq(:other)
     end
 
     it 'should cause punctuation not to be omitted if it is set to false' do
-      subject = described_class.new([:one, :other], {:clean => false})
+      classifier = Reclassifier::Bayes.new([:one, :other], {:clean => false})
 
-      subject.train(:one, '! ! ! ! bbb')
-      subject.train(:other, 'aaa')
+      classifier.train(:one, '! ! ! ! bbb')
+      classifier.train(:other, 'aaa')
 
-      subject.classify('! aaa !').should eq(:one)
+      classifier.classify('! aaa !').should eq(:one)
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reclassifier
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-24 00:00:00.000000000 Z
+date: 2013-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -91,6 +91,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Bayesian and Latent Semantic Indexing classification of text.
 email:
 - rroblak@gmail.com