otherinbox-classifier 1.3.1.1 → 1.3.1.2.20121218

data/lib/classifier/bayes.rb

@@ -28,6 +28,7 @@ class Bayes
 			@categories[category][word]      +=     count
 			@total_words += count
 		end
+    reset_correct_counts!
 	end
 
 	#
@@ -55,6 +56,7 @@ class Bayes
 				@total_words -= count
 			end
 		end
+    reset_correct_counts!
 	end
 		
 	#
@@ -80,24 +82,28 @@ class Bayes
     myclassify_with_word_hash(text.word_hash)
   end
 
-  # http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html
-  def myclassify_with_word_hash(word_hash)
+  def myclassify_with_word_hash(word_hash, debugging_info = nil)
     member_term_count = @categories[:Member].size
     nonmember_term_count = @categories[:"Not member"].size
+
     term_count = member_term_count + nonmember_term_count
     score = 0
     word_hash.each do |word, count|
       # count of words in each category
-      member_count = @categories[:Member][word].to_i + 1
-      nonmember_count = @categories[:"Not member"][word].to_i + 1
-      next if member_count.to_i == 1 && nonmember_count.to_i == 1      
+      member_count = @categories[:Member][word].to_i + 0.1
+      nonmember_count = @categories[:"Not member"][word].to_i + 0.1
+      next if member_count == 0.1 && nonmember_count == 0.1
 
       # find relative prob word is in class -- p(w|c)
-      word_member_p = (member_count) / (total_member_count + term_count).to_f
-      word_nonmember_p =  (nonmember_count) / (total_nonmember_count + term_count).to_f
+      word_member_p = (member_count) / (total_member_count_correct + term_count).to_f
+      word_nonmember_p =  (nonmember_count) / (total_nonmember_count_correct + term_count).to_f
       
       word_pr = Math.log(word_member_p / word_nonmember_p)
       score += word_pr * count
+      if debugging_info
+        debugging_info[word] = word_pr * count
+      end
+      #print "#{word_pr * count}: #{word}\n"
     end 
     if score > 0
       return "Member", score
@@ -155,10 +161,34 @@ class Bayes
 	def add_category(category)
 		@categories[category.prepare_category_name] = Hash.new
 	end
+
+  def remove_low_frequency_words(threshold = 5)
+    @categories.each do |_, word_counts|
+      word_counts.to_a.each do |word, count|
+        if count < threshold
+          word_counts.delete(word)
+        end
+      end
+    end
+    reset_correct_counts!
+  end
 	
 	alias append_category add_category
 
   private
+  def reset_correct_counts!
+    @total_member_count_correct = nil
+    @total_nonmember_count_correct = nil
+  end
+
+  def total_member_count_correct
+    @total_member_count_correct ||= @categories[:Member].values.inject(0) {|sum, element| sum+element}
+  end
+
+  def total_nonmember_count_correct
+    @total_nonmember_count_correct ||= @categories[:"Not member"].values.inject(0) {|sum, element| sum+element}
+  end
+
   def total_member_count
     @total_member_count ||= @categories[:Member].values.inject(0) {|sum, element| sum+element}
   end

data/lib/classifier/extensions/word_hash.rb

@@ -94,6 +94,7 @@ class String
       "dont",
       "ever",
       "first",
+      "for",
       "from",
       "have",
       "her",
@@ -150,5 +151,6 @@ class String
       "yes",
       "you",
       "youll",
+      "your"
       ].to_set
 end

metadata

@@ -1,85 +1,80 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: otherinbox-classifier
-version: !ruby/object:Gem::Version 
-  version: 1.3.1.1
+version: !ruby/object:Gem::Version
+  version: 1.3.1.2.20121218
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Lucas Carlson
 autorequire: classifier
 bindir: bin
-cert_chain: 
-date: 2008-01-19 00:00:00 -08:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+cert_chain: []
+date: 2008-01-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: stemmer
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
   type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.0.0
-    version: 
 description: A general classifier module to allow Bayesian and other types of classifications.
 email: lucas@rufy.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - lib/classifier.rb
-- lib/classifier
 - lib/classifier/bayes.rb
 - lib/classifier/lsi.rb
-- lib/classifier/extensions
 - lib/classifier/extensions/string.rb
 - lib/classifier/extensions/vector.rb
 - lib/classifier/extensions/vector_serialize.rb
 - lib/classifier/extensions/word_hash.rb
-- lib/classifier/lsi
 - lib/classifier/lsi/content_node.rb
 - lib/classifier/lsi/summary.rb
 - lib/classifier/lsi/word_list.rb
 - bin/bayes.rb
 - bin/summarize.rb
-- test/bayes
 - test/bayes/bayesian_test.rb
 - test/test_helper.rb
-- test/extensions
 - test/extensions/word_hash_test.rb
-- test/lsi
 - test/lsi/lsi_test.rb
 - README
 - Rakefile
 - LICENSE
-has_rdoc: true
 homepage: http://classifier.rufy.com/
+licenses: []
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
       version: 0.0.0
-  version: 
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
-requirements: 
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
 - A porter-stemmer module to split word stems.
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 1
 summary: A general classifier module to allow Bayesian and other types of classifications.
 test_files: []
-