despamilator 1.0 → 1.1

data/History.txt

@@ -46,3 +46,13 @@
 * Split out rails plugin (now despamilator-rails).
 
 * Added rdoc.
+
+=== 1.1
+
+* Added new filter to detect a cache busting technique.
+
+* Significantly increased aggressiveness of URI and HTML filters.
+
+* Added a heap of new spam and clean samples.
+
+* Added test string ("gtubs") that will score 100. For use in tests.

data/Manifest.txt

@@ -11,6 +11,7 @@ despamilator.gemspec
 lib/despamilator.rb
 lib/despamilator/filter.rb
 lib/despamilator/filter/funky_consonant.rb
+lib/despamilator/filter/gtubs_test_filter.rb
 lib/despamilator/filter/html_tags.rb
 lib/despamilator/filter/ip_address_url.rb
 lib/despamilator/filter/long_words.rb
@@ -20,12 +21,14 @@ lib/despamilator/filter/numbers_and_words.rb
 lib/despamilator/filter/script_tag.rb
 lib/despamilator/filter/shouting.rb
 lib/despamilator/filter/square_brackets.rb
+lib/despamilator/filter/trailing_number.rb
 lib/despamilator/filter/urls.rb
 lib/despamilator/filter_base.rb
 scripts/despamilator_score.rb
 spec/despamilator_spec.rb
 spec/filter_base_spec.rb
 spec/filters/funky_consonant_spec.rb
+spec/filters/gtubs_test_filter_spec.rb
 spec/filters/html_tags_spec.rb
 spec/filters/ip_address_url_spec.rb
 spec/filters/long_words_spec.rb
@@ -35,6 +38,7 @@ spec/filters/numbers_and_words_spec.rb
 spec/filters/script_tag_spec.rb
 spec/filters/shouting_spec.rb
 spec/filters/square_brackets_spec.rb
+spec/filters/trailing_number_spec.rb
 spec/filters/urls_spec.rb
 spec/helpers/corpus_helper.rb
 spec/helpers/filter_helper.rb

data/Rakefile

@@ -21,8 +21,7 @@ end
 require 'newgem/tasks'
 Dir['tasks/**/*.rake'].each { |t| load t }
 
-# TODO - want other tests/tasks run by default? Add them to the list
-# remove_task :default
+desc "Run tests"
 task :test => [:spec]
 task :default => [:test]
 task :install => [:install_gem]

data/despamilator.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{despamilator}
-  s.version = "1.0"
+  s.version = "1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Stephen Hardisty"]
-  s.date = %q{2010-12-11}
+  s.date = %q{2011-01-26}
   s.description = %q{Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances:
 Spam being submitted in my web forms and CAPTCHAS being intrusive. Despamilator will apply
 some commonly used heuristics from the world of anti-spam to help you decide whether your users are human or machine.}
   s.email = ["moowahaha@hotmail.com"]
   s.extra_rdoc_files = ["History.txt", "Manifest.txt", "PostInstall.txt"]
-  s.files = [".rspec", ".rvmrc", "Gemfile", "Gemfile.lock", "History.txt", "Manifest.txt", "PostInstall.txt", "README.rdoc", "Rakefile", "despamilator.gemspec", "lib/despamilator.rb", "lib/despamilator/filter.rb", "lib/despamilator/filter/funky_consonant.rb", "lib/despamilator/filter/html_tags.rb", "lib/despamilator/filter/ip_address_url.rb", "lib/despamilator/filter/long_words.rb", "lib/despamilator/filter/naughty_q.rb", "lib/despamilator/filter/naughty_words.rb", "lib/despamilator/filter/numbers_and_words.rb", "lib/despamilator/filter/script_tag.rb", "lib/despamilator/filter/shouting.rb", "lib/despamilator/filter/square_brackets.rb", "lib/despamilator/filter/urls.rb", "lib/despamilator/filter_base.rb", "scripts/despamilator_score.rb", "spec/despamilator_spec.rb", "spec/filter_base_spec.rb", "spec/filters/funky_consonant_spec.rb", "spec/filters/html_tags_spec.rb", "spec/filters/ip_address_url_spec.rb", "spec/filters/long_words_spec.rb", "spec/filters/naughty_q_spec.rb", "spec/filters/naughty_words_spec.rb", "spec/filters/numbers_and_words_spec.rb", "spec/filters/script_tag_spec.rb", "spec/filters/shouting_spec.rb", "spec/filters/square_brackets_spec.rb", "spec/filters/urls_spec.rb", "spec/helpers/corpus_helper.rb", "spec/helpers/filter_helper.rb", "spec/helpers/spec_helper.rb", "tasks/test.rake"]
+  s.files = [".rspec", ".rvmrc", "Gemfile", "Gemfile.lock", "History.txt", "Manifest.txt", "PostInstall.txt", "README.rdoc", "Rakefile", "despamilator.gemspec", "lib/despamilator.rb", "lib/despamilator/filter.rb", "lib/despamilator/filter/funky_consonant.rb", "lib/despamilator/filter/gtubs_test_filter.rb", "lib/despamilator/filter/html_tags.rb", "lib/despamilator/filter/ip_address_url.rb", "lib/despamilator/filter/long_words.rb", "lib/despamilator/filter/naughty_q.rb", "lib/despamilator/filter/naughty_words.rb", "lib/despamilator/filter/numbers_and_words.rb", "lib/despamilator/filter/script_tag.rb", "lib/despamilator/filter/shouting.rb", "lib/despamilator/filter/square_brackets.rb", "lib/despamilator/filter/trailing_number.rb", "lib/despamilator/filter/urls.rb", "lib/despamilator/filter_base.rb", "scripts/despamilator_score.rb", "spec/despamilator_spec.rb", "spec/filter_base_spec.rb", "spec/filters/funky_consonant_spec.rb", "spec/filters/gtubs_test_filter_spec.rb", "spec/filters/html_tags_spec.rb", "spec/filters/ip_address_url_spec.rb", "spec/filters/long_words_spec.rb", "spec/filters/naughty_q_spec.rb", "spec/filters/naughty_words_spec.rb", "spec/filters/numbers_and_words_spec.rb", "spec/filters/script_tag_spec.rb", "spec/filters/shouting_spec.rb", "spec/filters/square_brackets_spec.rb", "spec/filters/trailing_number_spec.rb", "spec/filters/urls_spec.rb", "spec/helpers/corpus_helper.rb", "spec/helpers/filter_helper.rb", "spec/helpers/spec_helper.rb", "tasks/test.rake"]
   s.homepage = %q{http://github.com/moowahaha/despamilator}
   s.post_install_message = %q{PostInstall.txt}
   s.rdoc_options = ["--main", "README.rdoc"]
@@ -26,11 +26,14 @@ some commonly used heuristics from the world of anti-spam to help you decide whe
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rubyforge>, [">= 2.0.4"])
       s.add_development_dependency(%q<hoe>, [">= 2.7.0"])
     else
+      s.add_dependency(%q<rubyforge>, [">= 2.0.4"])
       s.add_dependency(%q<hoe>, [">= 2.7.0"])
     end
   else
+    s.add_dependency(%q<rubyforge>, [">= 2.0.4"])
     s.add_dependency(%q<hoe>, [">= 2.7.0"])
   end
 end

data/lib/despamilator.rb

@@ -14,7 +14,7 @@ require 'despamilator/filter'
 #  dspam.matched_by #=> array of matching filters
 
 class Despamilator
-  VERSION = "1.0"
+  VERSION = "1.1"
 
   # Constructor. Takes the text you which to parse and score.
 

data/lib/despamilator/filter/gtubs_test_filter.rb

@@ -0,0 +1,21 @@
+require 'despamilator/filter_base'
+
+module DespamilatorFilter
+
+  class GtubsTestFilter < Despamilator::FilterBase
+
+    def name
+      'GTubs Test Filter'
+    end
+
+    def description
+      'Detects the special test string (Despamilator.gtubs_test_string) and assigns a big score.'
+    end
+
+    def parse text
+      self.append_score = 100 if text == Despamilator.gtubs_test_string
+    end
+
+  end
+
+end

data/lib/despamilator/filter/html_tags.rb

@@ -9,7 +9,7 @@ module DespamilatorFilter
 
       html_tags.each do |tag|
         if text.match(/<\s*#{tag}\W/) || text.match(/<\n*#{tag}\W/) || text.match(/\W#{tag}\s*\//) || text.match(/\W#{tag}\n*\//)
-          self.append_score = 0.3
+          self.append_score = 0.45
         end
       end
     end

data/lib/despamilator/filter/naughty_words.rb

@@ -39,6 +39,8 @@ module DespamilatorFilter
     cock
     pussy
     clit
+    preteen
+    lolita
    }
     end
 

data/lib/despamilator/filter/trailing_number.rb

@@ -0,0 +1,21 @@
+require 'despamilator/filter_base'
+
+module DespamilatorFilter
+
+  class TrailingNumber < Despamilator::FilterBase
+
+    def name
+      'Trailing Number'
+    end
+
+    def description
+      'Detects a trailing cache busting number'
+    end
+
+    def parse text
+      self.append_score = 0.1 if text =~ /\b\d+\s*$/
+    end
+
+  end
+
+end

data/lib/despamilator/filter/urls.rb

@@ -18,7 +18,7 @@ module DespamilatorFilter
       text.gsub!(/http:\/\/\d+\.\d+\.\d+\.\d+/, '')
       
       1.upto(text.scan(/http:\/\//).length) do
-        self.append_score = 0.20
+        self.append_score = 0.5
       end
     end
 

data/spec/filters/gtubs_test_filter_spec.rb

@@ -0,0 +1,9 @@
+describe DespamilatorFilter::GtubsTestFilter do
+  the_name_should_be 'GTubs Test Filter'
+  the_description_should_be 'Detects the special test string (Despamilator.gtubs_test_string) and assigns a big score.'
+
+  despamilator_should_apply_the_filter_for(Despamilator.gtubs_test_string)
+
+  a_single_match_of(Despamilator.gtubs_test_string, should_score: 100)
+  a_multiple_match_of(Despamilator.gtubs_test_string, should_score: [100, 1.times])
+end

data/spec/filters/html_tags_spec.rb

@@ -5,8 +5,8 @@ describe DespamilatorFilter::HtmlTags do
 
   despamilator_should_apply_the_filter_for('<xmp>')
 
-  a_single_match_of('<xmp>', should_score: 0.3)
-  a_multiple_match_of('<h1></h1> <h2></h2>', should_score: [0.6, 2.times])
+  a_single_match_of('<xmp>', should_score: 0.45)
+  a_multiple_match_of('<h1></h1> <h2></h2>', should_score: [0.9, 2.times])
 
   [
           '!--',
@@ -117,7 +117,7 @@ describe DespamilatorFilter::HtmlTags do
         it "should detect '#{tag}'" do
           dspam = DespamilatorFilter::HtmlTags.new
           dspam.parse(tag)
-          dspam.score.should == 0.3
+          dspam.score.should == 0.45
         end
 
       end

data/spec/filters/trailing_number_spec.rb

@@ -0,0 +1,10 @@
+describe DespamilatorFilter::TrailingNumber do
+
+  the_name_should_be 'Trailing Number'
+  the_description_should_be 'Detects a trailing cache busting number'
+
+  despamilator_should_apply_the_filter_for('hello 123 ')
+
+  a_single_match_of('hello 123', should_score: 0.1)
+
+end

data/spec/filters/urls_spec.rb

@@ -5,7 +5,7 @@ describe DespamilatorFilter::URLs do
 
   despamilator_should_apply_the_filter_for('zt')
 
-  a_single_match_of('http://www.blah.com', should_score: 0.2)
-  a_multiple_match_of('http://www.blah.com http://www.poop.com', should_score: [0.4, 2.times])
+  a_single_match_of('http://www.blah.com', should_score: 0.5)
+  a_multiple_match_of('http://www.blah.com http://www.poop.com', should_score: [1.0, 2.times])
 
 end

metadata

@@ -4,8 +4,8 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 0
-  version: "1.0"
+  - 1
+  version: "1.1"
 platform: ruby
 authors: 
 - Stephen Hardisty
@@ -13,7 +13,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-09 00:00:00 +11:00
+date: 2011-01-26 00:00:00 +11:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -74,6 +74,7 @@ files:
 - lib/despamilator.rb
 - lib/despamilator/filter.rb
 - lib/despamilator/filter/funky_consonant.rb
+- lib/despamilator/filter/gtubs_test_filter.rb
 - lib/despamilator/filter/html_tags.rb
 - lib/despamilator/filter/ip_address_url.rb
 - lib/despamilator/filter/long_words.rb
@@ -83,12 +84,14 @@ files:
 - lib/despamilator/filter/script_tag.rb
 - lib/despamilator/filter/shouting.rb
 - lib/despamilator/filter/square_brackets.rb
+- lib/despamilator/filter/trailing_number.rb
 - lib/despamilator/filter/urls.rb
 - lib/despamilator/filter_base.rb
 - scripts/despamilator_score.rb
 - spec/despamilator_spec.rb
 - spec/filter_base_spec.rb
 - spec/filters/funky_consonant_spec.rb
+- spec/filters/gtubs_test_filter_spec.rb
 - spec/filters/html_tags_spec.rb
 - spec/filters/ip_address_url_spec.rb
 - spec/filters/long_words_spec.rb
@@ -98,6 +101,7 @@ files:
 - spec/filters/script_tag_spec.rb
 - spec/filters/shouting_spec.rb
 - spec/filters/square_brackets_spec.rb
+- spec/filters/trailing_number_spec.rb
 - spec/filters/urls_spec.rb
 - spec/helpers/corpus_helper.rb
 - spec/helpers/filter_helper.rb