groupie 0.2.2 → 0.3.0

data/.gitignore

@@ -0,0 +1 @@
+*.gemspec

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0

data/lib/groupie.rb

@@ -12,16 +12,26 @@ class Groupie
     @groups[group] ||= Group.new(group)
   end
 
+  def unique_words
+    @unique_words ||= (
+      total_count = @groups.values.map {|group| group.word_counts}.inject{|total, counts| total.merge(counts){|key,o,n| o+n}}
+      median_index = [total_count.values.size * 3 / 4 - 1, 1].max
+      median_frequency = total_count.values.sort[median_index]
+      total_count.select{|word, count| count <= median_frequency}.map(&:first)
+    )
+  end
+
   def classify(entry, strategy=:sum)
     results = {}
     total_count = @groups.inject(0) do |sum, name_group|
       group = name_group.last
       count = group.count(entry)
-      if strategy==:sum
+      case strategy
+      when :sum
         sum += count
-      elsif strategy==:sqrt
+      when :sqrt, :unique
         sum += Math::sqrt(count)
-      elsif strategy==:log
+      when :log
         sum += Math::log10(count) if count > 0
       else
         raise "Invalid strategy: #{strategy}"
@@ -32,11 +42,12 @@ class Groupie
 
     @groups.each do |name, group|
       count = group.count(entry)
-      if strategy==:sum
+      case strategy
+      when :sum
         # keep count
-      elsif strategy==:sqrt
+      when :sqrt, :unique
         count = Math::sqrt(count)
-      elsif strategy==:log
+      when :log
         count = Math::log10(count) if count > 0
       else
         raise "Invalid strategy: #{strategy}"
@@ -49,6 +60,9 @@ class Groupie
   # Classify a text by taking the average of all word classifications.
   def classify_text(words, strategy=:sum)
     hits = 0
+    if strategy==:unique
+      words = words & unique_words
+    end
     group_score_sums = words.inject({}) do |results, word|
       word_results = classify(word, strategy)
       next results if word_results.empty?

data/lib/groupie/group.rb

@@ -1,10 +1,11 @@
 class Groupie
   class Group
+    attr_reader :word_counts
     def initialize(name)
       @name = name
       @word_counts = {}
     end
-    
+
     def words
       @word_counts.keys
     end
@@ -16,6 +17,7 @@ class Groupie
       end
       nil
     end
+    alias << add
 
     # Return the count for a specific +word+.
     def count(word)

data/readme.rdoc

@@ -4,11 +4,7 @@ Groupie is a simple way to group texts and classify new texts as being a likely
 
 The eventual goal is to have Groupie work as a sort of bayesian spam filter, where you feed it spam and ham (non-spam) and ask it to classify new texts as spam or ham. Applications for this are e-mail spam filtering and blog spam filtering. Other sorts of categorizing might be interesting as well, such as finding suitable tags for a blog post or bookmark.
 
-== Goals 
-
-Groupie is a 'fun' project that has the following goals, in descending order of importance:
-* Have fun playing with code
-* Play with Bayesian-like (spam) filtering
+Started and forgotten in 2009 as a short-lived experiment, in 2010 Groupie got new features when I started using it on a RSS reader project that classified news items into "Interesting" and "Not interesting" categories.
 
 == Current functionality
 
@@ -19,6 +15,7 @@ Current funcionality includes:
 * Add texts (as an Array of Strings) to any number of groups.
 * Classify a single word to check the likelihood it belongs to each group.
 * Do classification for complete (tokenized) texts.
+* Pick classification strategy to weigh repeat words differently (weigh by sum, square root or log10 of words in group)
 
 == License
 

data/spec/groupie/group_spec.rb

@@ -2,6 +2,31 @@ require File.join(File.dirname(__FILE__), %w[.. spec_helper])
 require 'yaml'
 
 describe Groupie::Group do
+  describe "add" do
+    before(:each) do
+      @group = Groupie::Group.new("test")
+    end
+
+    it "should accept a single string" do
+      @group.add "bla"
+      @group.words.should == %w[bla]
+    end
+
+    it "should accept an Array of strings" do
+      @group.add ["bla", "bla2"]
+      @group.words.should == %w[bla bla2]
+    end
+
+    it "should accept multiple strings" do
+      @group.add "bla", "bla2"
+      @group.words.should == %w[bla bla2]
+    end
+
+    it "should be aliased as <<" do
+      @group.method(:add).should == @group.method(:<<)
+    end
+  end
+
   it "can be serialized and loaded through YAML" do
     group = Groupie::Group.new 'group'
     group.add %w[buy flowers]

data/spec/groupie_spec.rb

@@ -73,6 +73,32 @@ describe Groupie do
           g.classify('word', :log).should == {:spam=>0.75, :ham=>0.25}
         end
       end
+
+      describe "unique" do
+        it "should should behave as sqrt strategy" do
+          g = Groupie.new
+          g[:spam].add %w[buy viagra now]
+          g[:ham].add %w[buy flowers now]
+          g.classify('buy', :unique).should == g.classify('buy', :sqrt)
+          g.classify('flowers', :unique).should == g.classify('flowers', :sqrt)
+        end
+      end
+    end
+  end
+
+  describe "unique_words" do
+    it "should exclude all words in the 4th quintile of all groups" do
+      g = Groupie.new
+      g[:spam].add %w[one two two three three three four four four four]
+      g[:ham].add %w[apple banana pear orange three]
+      g.unique_words.sort.should == %w[one two apple banana pear orange].sort
+    end
+
+    it "should work on an empty word set" do
+      g = Groupie.new
+      g[:spam].add []
+      g[:ham].add []
+      g.unique_words.should == []
     end
   end
 
@@ -108,7 +134,7 @@ describe Groupie do
       g[:ham].add %w[buy flowers now]
       g.classify_text(%w[buy buckets now]).should == {:spam=>0.5, :ham=>0.5}
     end
-    
+
     it "should support the sqrt strategy" do
       g = Groupie.new
       g[:spam].add %w[one] * 9
@@ -117,7 +143,7 @@ describe Groupie do
       g[:ham].add %w[two]
       g.classify_text(%w[one two three], :sqrt).should == {:spam=>0.75, :ham=>0.25}
     end
-    
+
     it "should support the log strategy" do
       g = Groupie.new
       g[:spam].add %w[one] * 100
@@ -126,5 +152,12 @@ describe Groupie do
       g[:ham].add %w[two] * 100
       g.classify_text(%w[one two three], :log).should == {:spam=>0.5, :ham=>0.5}
     end
+
+    it "should only rate unique words for the unique strategy" do
+      g = Groupie.new
+      g[:spam].add %w[one two two three three three four four four four]
+      g[:ham].add %w[apple banana pear]
+      g.classify_text(%w[one two three apple banana], :unique).should == {:spam=>0.5, :ham=>0.5}
+    end
   end
 end

metadata

@@ -5,9 +5,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 2
-  version: 0.2.2
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Wes Oldenbeuving
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-25 00:00:00 +02:00
+date: 2010-07-29 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -42,10 +42,10 @@ extra_rdoc_files:
 - LICENSE
 files: 
 - .document
+- .gitignore
 - LICENSE
 - Rakefile
 - VERSION
-- groupie.gemspec
 - lib/groupie.rb
 - lib/groupie/core_ext/string.rb
 - lib/groupie/group.rb

data/groupie.gemspec

@@ -1,63 +0,0 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |s|
-  s.name = %q{groupie}
-  s.version = "0.2.2"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Wes Oldenbeuving"]
-  s.date = %q{2010-07-25}
-  s.description = %q{Group and classify text based on likelyhood of being included in a text of a specific category}
-  s.email = %q{narnach@gmail.com}
-  s.extra_rdoc_files = [
-    "LICENSE"
-  ]
-  s.files = [
-    ".document",
-     "LICENSE",
-     "Rakefile",
-     "VERSION",
-     "groupie.gemspec",
-     "lib/groupie.rb",
-     "lib/groupie/core_ext/string.rb",
-     "lib/groupie/group.rb",
-     "readme.rdoc",
-     "spec/fixtures/ham/email_ham1.txt",
-     "spec/fixtures/ham/spam.la-44116217.txt",
-     "spec/fixtures/spam/email_spam1.txt",
-     "spec/fixtures/spam/email_spam2.txt",
-     "spec/fixtures/spam/spam.la-44118014.txt",
-     "spec/groupie/core_ext/string_spec.rb",
-     "spec/groupie/group_spec.rb",
-     "spec/groupie_spec.rb",
-     "spec/spec_helper.rb"
-  ]
-  s.homepage = %q{http://github.com/Narnach/groupie}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Group and classify text}
-  s.test_files = [
-    "spec/groupie/core_ext/string_spec.rb",
-     "spec/groupie/group_spec.rb",
-     "spec/groupie_spec.rb",
-     "spec/spec_helper.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<testy>, [">= 0"])
-    else
-      s.add_dependency(%q<testy>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<testy>, [">= 0"])
-  end
-end
-