despamilator 1.1 → 2.0

data/despamilator.gemspec

@@ -2,27 +2,26 @@
 
 Gem::Specification.new do |s|
   s.name = %q{despamilator}
-  s.version = "1.1"
+  s.version = "2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Stephen Hardisty"]
-  s.date = %q{2011-01-26}
+  s.date = %q{2011-05-24}
   s.description = %q{Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances:
 Spam being submitted in my web forms and CAPTCHAS being intrusive. Despamilator will apply
 some commonly used heuristics from the world of anti-spam to help you decide whether your users are human or machine.}
   s.email = ["moowahaha@hotmail.com"]
-  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "PostInstall.txt"]
-  s.files = [".rspec", ".rvmrc", "Gemfile", "Gemfile.lock", "History.txt", "Manifest.txt", "PostInstall.txt", "README.rdoc", "Rakefile", "despamilator.gemspec", "lib/despamilator.rb", "lib/despamilator/filter.rb", "lib/despamilator/filter/funky_consonant.rb", "lib/despamilator/filter/gtubs_test_filter.rb", "lib/despamilator/filter/html_tags.rb", "lib/despamilator/filter/ip_address_url.rb", "lib/despamilator/filter/long_words.rb", "lib/despamilator/filter/naughty_q.rb", "lib/despamilator/filter/naughty_words.rb", "lib/despamilator/filter/numbers_and_words.rb", "lib/despamilator/filter/script_tag.rb", "lib/despamilator/filter/shouting.rb", "lib/despamilator/filter/square_brackets.rb", "lib/despamilator/filter/trailing_number.rb", "lib/despamilator/filter/urls.rb", "lib/despamilator/filter_base.rb", "scripts/despamilator_score.rb", "spec/despamilator_spec.rb", "spec/filter_base_spec.rb", "spec/filters/funky_consonant_spec.rb", "spec/filters/gtubs_test_filter_spec.rb", "spec/filters/html_tags_spec.rb", "spec/filters/ip_address_url_spec.rb", "spec/filters/long_words_spec.rb", "spec/filters/naughty_q_spec.rb", "spec/filters/naughty_words_spec.rb", "spec/filters/numbers_and_words_spec.rb", "spec/filters/script_tag_spec.rb", "spec/filters/shouting_spec.rb", "spec/filters/square_brackets_spec.rb", "spec/filters/trailing_number_spec.rb", "spec/filters/urls_spec.rb", "spec/helpers/corpus_helper.rb", "spec/helpers/filter_helper.rb", "spec/helpers/spec_helper.rb", "tasks/test.rake"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "PostInstall.txt", "conf/unusual_characters.txt"]
+  s.files = [".rspec", ".rvmrc", "Gemfile", "Gemfile.lock", "History.txt", "Manifest.txt", "PostInstall.txt", "README.rdoc", "Rakefile", "conf/unusual_characters.txt", "despamilator.gemspec", "lib/despamilator.rb", "lib/despamilator/filter.rb", "lib/despamilator/filter/gtubs_test_filter.rb", "lib/despamilator/filter/html_tags.rb", "lib/despamilator/filter/ip_address_url.rb", "lib/despamilator/filter/long_words.rb", "lib/despamilator/filter/naughty_words.rb", "lib/despamilator/filter/numbers_and_words.rb", "lib/despamilator/filter/script_tag.rb", "lib/despamilator/filter/shouting.rb", "lib/despamilator/filter/square_brackets.rb", "lib/despamilator/filter/trailing_number.rb", "lib/despamilator/filter/unusual_characters.rb", "lib/despamilator/filter/urls.rb", "lib/despamilator/filter_base.rb", "scripts/despamilator_score.rb", "scripts/from_file.rb", "spec/despamilator_spec.rb", "spec/filter_base_spec.rb", "spec/filters/gtubs_test_filter_spec.rb", "spec/filters/html_tags_spec.rb", "spec/filters/ip_address_url_spec.rb", "spec/filters/long_words_spec.rb", "spec/filters/naughty_words_spec.rb", "spec/filters/numbers_and_words_spec.rb", "spec/filters/script_tag_spec.rb", "spec/filters/shouting_spec.rb", "spec/filters/square_brackets_spec.rb", "spec/filters/trailing_number_spec.rb", "spec/filters/unusual_characters_spec.rb", "spec/filters/urls_spec.rb", "spec/helpers/corpus_helper.rb", "spec/helpers/filter_helper.rb", "spec/helpers/spec_helper.rb", "tasks/test.rake"]
   s.homepage = %q{http://github.com/moowahaha/despamilator}
   s.post_install_message = %q{PostInstall.txt}
   s.rdoc_options = ["--main", "README.rdoc"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{despamilator}
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.5.2}
   s.summary = %q{Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances: Spam being submitted in my web forms and CAPTCHAS being intrusive}
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/lib/despamilator.rb

@@ -14,7 +14,7 @@ require 'despamilator/filter'
 #  dspam.matched_by #=> array of matching filters
 
 class Despamilator
-  VERSION = "1.1"
+  VERSION = "2.0"
 
   # Constructor. Takes the text you which to parse and score.
 

data/lib/despamilator/filter/html_tags.rb

@@ -9,7 +9,7 @@ module DespamilatorFilter
 
       html_tags.each do |tag|
         if text.match(/<\s*#{tag}\W/) || text.match(/<\n*#{tag}\W/) || text.match(/\W#{tag}\s*\//) || text.match(/\W#{tag}\n*\//)
-          self.append_score = 0.45
+          self.append_score = 0.6
         end
       end
     end

data/lib/despamilator/filter/naughty_words.rb

@@ -16,12 +16,13 @@ module DespamilatorFilter
       text.downcase!
 
       naughty_words.each do |word|
-        self.append_score = 0.1 if text =~ /\b#{word}\b/
+        self.append_score = 0.1 if text =~ /\b#{word}s?\b/
       end
     end
 
     def naughty_words
       %w{
+    underage
     penis
     viagra
     bondage
@@ -30,10 +31,9 @@ module DespamilatorFilter
     shit
     dick
     tits
-    sex
     nude
     dicks
-    shemales
+    shemale
     dildo
     porn
     cock

data/lib/despamilator/filter/unusual_characters.rb

@@ -0,0 +1,47 @@
+require 'despamilator/filter_base'
+
+module DespamilatorFilter
+
+  class UnusualCharacters < Despamilator::FilterBase
+
+    def name
+      'Unusual Characters'
+    end
+
+    def description
+      'Detects and scores each occurrence of an unusual 2 or 3 character combination'
+    end
+
+    def parse text
+      initialize_combos
+      tokenize(text).each do |token|
+        self.append_score = 0.05 if @@combos[token.to_sym]
+      end
+    end
+
+    private
+
+    def tokenize text
+      tokens = []
+      text.downcase.split(/[^a-z]/).each do |word|
+        word.chars.each_with_index do |c, i|
+          substr = word[i,i+3]
+          tokens << substr.to_sym if substr.length == 3
+          tokens << substr[0,2].to_sym if substr.length > 1
+        end
+      end
+      tokens
+    end
+
+    def initialize_combos
+      @@combos ||= {}
+      return @@combos unless @@combos.empty?
+
+      File.open(File.join(File.dirname(__FILE__), %w{.. .. .. conf unusual_characters.txt}), 'r').each do |line|
+        @@combos[line.strip.to_sym] = true
+      end
+    end
+
+  end
+
+end

data/lib/despamilator/filter/urls.rb

@@ -16,9 +16,9 @@ module DespamilatorFilter
       text.downcase!
 
       text.gsub!(/http:\/\/\d+\.\d+\.\d+\.\d+/, '')
-      
+
       1.upto(text.scan(/http:\/\//).length) do
-        self.append_score = 0.5
+        self.append_score = 0.4
       end
     end
 

data/scripts/from_file.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+require 'cgi'
+
+slots = {};
+
+1.upto(10000) do |i|
+    slots[i] = true
+end
+
+dir = './spec/spam_corpus/'
+
+Dir[dir + '*.gz'].each do |file|
+    slots.delete(file.scan(/\d+/).first.to_i)
+end
+
+slots = slots.keys.sort
+
+File.open(ARGV[0] || raise).each do |line|
+    txt = dir + "#{slots.shift}.txt"
+    File.open(txt, 'w') do |fh|
+        fh.puts CGI.unescapeHTML(line)
+    end
+
+    `gzip #{txt}`
+end

data/spec/filters/html_tags_spec.rb

@@ -5,8 +5,8 @@ describe DespamilatorFilter::HtmlTags do
 
   despamilator_should_apply_the_filter_for('<xmp>')
 
-  a_single_match_of('<xmp>', should_score: 0.45)
-  a_multiple_match_of('<h1></h1> <h2></h2>', should_score: [0.9, 2.times])
+  a_single_match_of('<xmp>', should_score: 0.6)
+  a_multiple_match_of('<h1></h1> <h2></h2>', should_score: [1.2, 2.times])
 
   [
           '!--',
@@ -117,7 +117,7 @@ describe DespamilatorFilter::HtmlTags do
         it "should detect '#{tag}'" do
           dspam = DespamilatorFilter::HtmlTags.new
           dspam.parse(tag)
-          dspam.score.should == 0.45
+          dspam.score.should == 0.6
         end
 
       end

data/spec/filters/unusual_characters_spec.rb

@@ -0,0 +1,9 @@
+describe DespamilatorFilter::UnusualCharacters do
+  the_name_should_be 'Unusual Characters'
+  the_description_should_be 'Detects and scores each occurrence of an unusual 2 or 3 character combination'
+
+  despamilator_should_apply_the_filter_for('sx')
+
+  a_single_match_of('sx', should_score: 0.05)
+  a_multiple_match_of('sxsx', should_score: [0.1, 2.times])
+end

data/spec/filters/urls_spec.rb

@@ -3,9 +3,9 @@ describe DespamilatorFilter::URLs do
   the_name_should_be 'URLs'
   the_description_should_be 'Detects each url in a string'
 
-  despamilator_should_apply_the_filter_for('zt')
+  despamilator_should_apply_the_filter_for('http://www.blah.com')
 
-  a_single_match_of('http://www.blah.com', should_score: 0.5)
-  a_multiple_match_of('http://www.blah.com http://www.poop.com', should_score: [1.0, 2.times])
+  a_single_match_of('http://www.blah.com', should_score: 0.4)
+  a_multiple_match_of('http://www.blah.com http://www.poop.com', should_score: [0.8, 2.times])
 
 end

metadata

@@ -1,66 +1,57 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: despamilator
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 1
-  version: "1.1"
+version: !ruby/object:Gem::Version
+  version: '2.0'
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Stephen Hardisty
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-01-26 00:00:00 +11:00
+date: 2011-05-24 00:00:00.000000000 +10:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rubyforge
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2730490 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 0
-        - 4
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 2.0.4
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: hoe
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2730490
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: &2730250 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 7
-        - 0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 2.7.0
   type: :development
-  version_requirements: *id002
-description: |-
-  Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances:
-  Spam being submitted in my web forms and CAPTCHAS being intrusive. Despamilator will apply
-  some commonly used heuristics from the world of anti-spam to help you decide whether your users are human or machine.
-email: 
+  prerelease: false
+  version_requirements: *2730250
+description: ! 'Despamilator is a plugin based spam detector designed for use on your
+  web forms borne out of two annoyances:
+
+  Spam being submitted in my web forms and CAPTCHAS being intrusive. Despamilator
+  will apply
+
+  some commonly used heuristics from the world of anti-spam to help you decide whether
+  your users are human or machine.'
+email:
 - moowahaha@hotmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - History.txt
 - Manifest.txt
 - PostInstall.txt
-files: 
+- conf/unusual_characters.txt
+files:
 - .rspec
 - .rvmrc
 - Gemfile
@@ -70,38 +61,38 @@ files:
 - PostInstall.txt
 - README.rdoc
 - Rakefile
+- conf/unusual_characters.txt
 - despamilator.gemspec
 - lib/despamilator.rb
 - lib/despamilator/filter.rb
-- lib/despamilator/filter/funky_consonant.rb
 - lib/despamilator/filter/gtubs_test_filter.rb
 - lib/despamilator/filter/html_tags.rb
 - lib/despamilator/filter/ip_address_url.rb
 - lib/despamilator/filter/long_words.rb
-- lib/despamilator/filter/naughty_q.rb
 - lib/despamilator/filter/naughty_words.rb
 - lib/despamilator/filter/numbers_and_words.rb
 - lib/despamilator/filter/script_tag.rb
 - lib/despamilator/filter/shouting.rb
 - lib/despamilator/filter/square_brackets.rb
 - lib/despamilator/filter/trailing_number.rb
+- lib/despamilator/filter/unusual_characters.rb
 - lib/despamilator/filter/urls.rb
 - lib/despamilator/filter_base.rb
 - scripts/despamilator_score.rb
+- scripts/from_file.rb
 - spec/despamilator_spec.rb
 - spec/filter_base_spec.rb
-- spec/filters/funky_consonant_spec.rb
 - spec/filters/gtubs_test_filter_spec.rb
 - spec/filters/html_tags_spec.rb
 - spec/filters/ip_address_url_spec.rb
 - spec/filters/long_words_spec.rb
-- spec/filters/naughty_q_spec.rb
 - spec/filters/naughty_words_spec.rb
 - spec/filters/numbers_and_words_spec.rb
 - spec/filters/script_tag_spec.rb
 - spec/filters/shouting_spec.rb
 - spec/filters/square_brackets_spec.rb
 - spec/filters/trailing_number_spec.rb
+- spec/filters/unusual_characters_spec.rb
 - spec/filters/urls_spec.rb
 - spec/helpers/corpus_helper.rb
 - spec/helpers/filter_helper.rb
@@ -110,35 +101,30 @@ files:
 has_rdoc: true
 homepage: http://github.com/moowahaha/despamilator
 licenses: []
-
 post_install_message: PostInstall.txt
-rdoc_options: 
+rdoc_options:
 - --main
 - README.rdoc
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: despamilator
-rubygems_version: 1.3.7
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
-summary: "Despamilator is a plugin based spam detector designed for use on your web forms borne out of two annoyances: Spam being submitted in my web forms and CAPTCHAS being intrusive"
+summary: ! 'Despamilator is a plugin based spam detector designed for use on your
+  web forms borne out of two annoyances: Spam being submitted in my web forms and
+  CAPTCHAS being intrusive'
 test_files: []
-

data/lib/despamilator/filter/funky_consonant.rb

@@ -1,31 +0,0 @@
-require 'despamilator/filter_base'
-
-module DespamilatorFilter
-
-  class FunkyConsonant < Despamilator::FilterBase
-
-    def name
-      'Funky Consonant'
-    end
-
-    def description
-      'Detects and scores each occurrence of a consonant next to an unlikely character'
-    end
-
-    def parse text
-      text.downcase!
-
-      consonant_pairs.each do |pair|
-        [pair, pair.reverse].each do |combo_pair|
-          self.append_score = 0.05 unless text.scan(/#{combo_pair}/).empty?
-        end
-      end
-    end
-
-    def consonant_pairs
-      %w{ zt gb vk vt jk mj dm jm xz bn }
-    end
-    
-  end
-
-end

data/lib/despamilator/filter/naughty_q.rb

@@ -1,31 +0,0 @@
-require 'despamilator/filter_base'
-
-module DespamilatorFilter
-
-  class NaughtyQ < Despamilator::FilterBase
-
-    def name
-      'Naughty Q'
-    end
-
-    def description
-      'Detects possible misuse of the letter Q (English language)'
-    end
-
-    def parse text
-      post_matches = text.downcase.scan(/q(\w|\d)/)
-      pre_matches = text.downcase.scan(/(\w|\d)q/)
-
-      matches = post_matches + pre_matches
-
-      return unless matches
-
-      matches.each do |match|
-        match = match.first
-        self.append_score = 0.2 unless match == 'u' or match == 'a' or match == 'k'
-      end
-    end
-
-  end
-
-end